| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "log" |
| 10 | "math/rand/v2" |
| 11 | "os" |
| 12 | "slices" |
| 13 | "strings" |
| 14 | "sync" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/appconfig" |
| 18 | "github.com/carlosframework/platform/internal/hostclaim" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | |
| 22 | "github.com/carlosframework/platform/internal/domains" |
| 23 | ) |
| 24 | |
| 25 | // The multi-region ingress spec (2026-08-20, Part 3, closes the design half |
| 26 | // of #238): every platform box publishes what it homes so any edge can fold |
| 27 | // the fleet's per-box files into a deployment-wide host→home map. v1 writes |
| 28 | // only `role: "home"` — the anticipated read-local role values are Task 3's |
| 29 | // concern, not this file's. |
| 30 | // |
| 31 | // edgesFilePrefix is the flat prefix those files live under. It is named |
| 32 | // rather than spelled twice because there is now a READER on this side too — |
| 33 | // steering_loop.go lists it to fold the fleet's files (internal/edge's |
| 34 | // homeMap has its own copy, on the far side of an import boundary this |
| 35 | // package does not cross). |
| 36 | const edgesFilePrefix = "control/edges/" |
| 37 | |
| 38 | func edgesFileKey(box string) string { return edgesFilePrefix + box + ".json" } |
| 39 | |
| 40 | // edgeHost is the per-host entry in edgesFile.Hosts. An object, not a bare |
| 41 | // string, is deliberately load-bearing: it is read-local anticipation — |
| 42 | // Task 3's mirror and later roles (e.g. a "read" role) need somewhere to |
| 43 | // grow into without a wire-shape break. |
| 44 | type edgeHost struct { |
| 45 | Role string `json:"role"` |
| 46 | // Platform marks a row the platform minted (registry Platform or |
| 47 | // Provisioned) — the class edge.currentHosts admits to wildcard-parent |
| 48 | // derivation. Additive; always emitted (never omitempty), so a file with |
| 49 | // no "platform" key at all is definitely a pre-mark publisher, not a box |
| 50 | // homing nothing platform-class. A reader seeing no flag treats the host |
| 51 | // as unclassed and derives no parent from it (#300). |
| 52 | Platform bool `json:"platform"` |
| 53 | // Alias marks an attached custom domain (route_aliases, #287) as opposed |
| 54 | // to a registry route. Additive (omitempty). The reader's fold lets a |
| 55 | // route beat an alias on a name collision across boxes: a tenant |
| 56 | // attaching an operator's hostname must never capture it fleet-wide. |
| 57 | Alias bool `json:"alias,omitempty"` |
| 58 | // Replica marks the account's checkout replica for this host (#162): |
| 59 | // this box serves the account from a one-way mirror of it — same app, |
| 60 | // same account, role=replica record — instead of holding the |
| 61 | // authoritative data. Additive (omitempty), so every file written |
| 62 | // before the field existed parses byte-identically, and a reader that |
| 63 | // has never heard of it ignores it. Published from the HOME route's |
| 64 | // Replicas column (registry.Route.Replicas): a replica placement is |
| 65 | // where the account is SERVED, and the home's row is the one that |
| 66 | // knows. Read by the steering converger (a replica placement earns a |
| 67 | // latency record) and available to every edges-file reader; the edge's |
| 68 | // own dispatch never acts on it — a replica box serves its row exactly |
| 69 | // as any home serves its own, and a replica never hairpins. |
| 70 | Replica bool `json:"replica,omitempty"` |
| 71 | // Wildcard marks a delegated customer wildcard parent (wildcard_domains, |
| 72 | // #305): the key is "*.<parent>" and every edge should load the parent's |
| 73 | // mirrored wildcard PEM. With role "home" the box also serves the |
| 74 | // catch-all (dial target); with role "wildcard" it is the cert half |
| 75 | // alone. Additive (omitempty); never platform. |
| 76 | Wildcard bool `json:"wildcard,omitempty"` |
| 77 | // Account and App name the one app that owns everything this box serves |
| 78 | // under the hostname (#175 P7). The edge-local door on another box checks |
| 79 | // a door key against them before it backhauls here. Published only when |
| 80 | // every route behind the name agrees: the row itself, every path-prefix |
| 81 | // row sharing it, and for an alias or a served catch-all, its primary's. |
| 82 | // A hostname split between apps publishes neither, and the door refuses |
| 83 | // it remotely. Additive (omitempty). |
| 84 | Account string `json:"account,omitempty"` |
| 85 | App string `json:"app,omitempty"` |
| 86 | // Stamp is the claim's holder.stamp for the installation this box is |
| 87 | // serving (region HA spec 1 HA4). 0 = unstamped: no claim, or a writer |
| 88 | // that predates it. Two files claiming one host are settled by the |
| 89 | // higher stamp on the edge (homemap.go foldHomes). |
| 90 | Stamp int64 `json:"stamp,omitempty"` |
| 91 | } |
| 92 | |
| 93 | // hostOwner folds the owners of every route served under one hostname. Any |
| 94 | // disagreement, or any route with no account or app, leaves no owner. |
| 95 | type hostOwner struct { |
| 96 | account, app string |
| 97 | set, mixed bool |
| 98 | } |
| 99 | |
| 100 | func (o *hostOwner) add(account, app string) { |
| 101 | switch { |
| 102 | case account == "" || app == "": |
| 103 | o.mixed = true |
| 104 | case !o.set: |
| 105 | o.account, o.app, o.set = account, app, true |
| 106 | case o.account != account || o.app != app: |
| 107 | o.mixed = true |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | func (o hostOwner) owner() (account, app string) { |
| 112 | if !o.set || o.mixed { |
| 113 | return "", "" |
| 114 | } |
| 115 | return o.account, o.app |
| 116 | } |
| 117 | |
| 118 | // servedOwners answers, for each served row's hostname, the owner the edge's |
| 119 | // own dispatch would reach under it: the row's route, and the route each |
| 120 | // path-prefix row on it targets. A target that names a route in the registry |
| 121 | // counts whatever that route's status. When it is not ready the edge falls |
| 122 | // back to the host's own route today, but it counts anyway, so the owner never |
| 123 | // outlives the target turning ready. A target naming no route here can only |
| 124 | // fall back to the host's own route, which is already counted. |
| 125 | func servedOwners(routes []registry.Route, served map[string]edgeHost, paths []registry.RoutePath) map[string]*hostOwner { |
| 126 | byHost := make(map[string]registry.Route, len(routes)) |
| 127 | for _, r := range routes { |
| 128 | byHost[r.Host] = r |
| 129 | } |
| 130 | owners := make(map[string]*hostOwner, len(served)) |
| 131 | for host := range served { |
| 132 | r := byHost[host] |
| 133 | o := &hostOwner{} |
| 134 | o.add(release.RouteAccount(r), r.App) |
| 135 | owners[host] = o |
| 136 | } |
| 137 | for _, p := range paths { |
| 138 | o, ok := owners[p.Host] |
| 139 | if !ok { |
| 140 | continue |
| 141 | } |
| 142 | if t, ok := byHost[p.Target]; ok { |
| 143 | o.add(release.RouteAccount(t), t.App) |
| 144 | } |
| 145 | } |
| 146 | return owners |
| 147 | } |
| 148 | |
| 149 | // edgeHeadroom is this box's spare capacity at the time the file was |
| 150 | // written. Zero values mean "could not read", never "none left": a chooser |
| 151 | // ranks an unreadable box last, it does not exclude it. |
| 152 | type edgeHeadroom struct { |
| 153 | MemFreeMiB int64 `json:"mem_free_mib"` |
| 154 | DiskFreeMiB int64 `json:"disk_free_mib"` |
| 155 | Awake int `json:"awake"` |
| 156 | // MemTotalMiB and DiskTotalMiB are what the free figures are shares |
| 157 | // of (region HA spec 2 §7). Additive and omitted when 0, a reading |
| 158 | // that failed: a reader takes absent and 0 alike, as unknown. |
| 159 | MemTotalMiB int64 `json:"mem_total_mib,omitempty"` |
| 160 | DiskTotalMiB int64 `json:"disk_total_mib,omitempty"` |
| 161 | } |
| 162 | |
| 163 | const capRegionHA1 = "region-ha-1" |
| 164 | |
| 165 | // edgesFileCapabilities publishes region-ha-1 only when CARLOS_REGION_HA is |
| 166 | // set to something other than "" or "0". offline-move-1 is added per pass |
| 167 | // by withOfflineCapability. |
| 168 | func edgesFileCapabilities(getenv func(string) string) []string { |
| 169 | if !regionHAEnabled(getenv) { |
| 170 | return nil |
| 171 | } |
| 172 | return []string{capRegionHA1} |
| 173 | } |
| 174 | |
| 175 | // withOfflineCapability is caps with offline-move-1 and offline-stage-1 when |
| 176 | // capable (a box that can take part in an offline move can stage one). The loop |
| 177 | // passes armedOfflineCapable(): the armed moves worker's own answer, the one |
| 178 | // the reconcile hooks act on, so a box never publishes the capability |
| 179 | // without the hold that makes it safe (a target that published it with no |
| 180 | // armed worker would mint a landing row ready). caps is not modified. |
| 181 | func withOfflineCapability(caps []string, capable bool) []string { |
| 182 | if !capable { |
| 183 | return caps |
| 184 | } |
| 185 | return append(slices.Clone(caps), capOfflineMove1, capOfflineStage1) |
| 186 | } |
| 187 | |
| 188 | // headroomCapabilities is what headroomFor is asked about: region HA's |
| 189 | // capability alone. offline-move-1 publishes no headroom, so a box that |
| 190 | // gains only it publishes the file it always has, plus the one string. |
| 191 | func headroomCapabilities(caps []string) []string { |
| 192 | if slices.Contains(caps, capRegionHA1) { |
| 193 | return caps |
| 194 | } |
| 195 | return nil |
| 196 | } |
| 197 | |
| 198 | // awakeCount is how many routes this box has running now. An error reads |
| 199 | // as zero: headroom is advisory. |
| 200 | func awakeCount(reg *registry.Registry) int { |
| 201 | routes, err := reg.List() |
| 202 | if err != nil { |
| 203 | return 0 |
| 204 | } |
| 205 | n := 0 |
| 206 | for _, r := range routes { |
| 207 | if r.Status == "ready" && !r.Hibernate { |
| 208 | n++ |
| 209 | } |
| 210 | } |
| 211 | return n |
| 212 | } |
| 213 | |
| 214 | // edgesFile is the wire shape at edgesFileKey(box), schema v1 (spec Part 3): |
| 215 | // |
| 216 | // {"v":1,"box":"quiet-harbour","eip":"203.0.113.10","region":"eu-west-1","generation":"2026-08-22T09:00:00Z","hosts":{"app.example.com":{"role":"home"}}} |
| 217 | // |
| 218 | // Region is additive (Part 3, Task 4's steering converger needs it for |
| 219 | // latency records): omitempty, absent entirely when it could not be |
| 220 | // resolved (see edgesFileRegion) — a pre-region body written by an older |
| 221 | // build still parses, and this build's own body for an unresolved-region |
| 222 | // box is byte-identical to what it always published. |
| 223 | // |
| 224 | // The value is the AWS region (e.g. "eu-west-1"), NOT CARLOS_REGION — that |
| 225 | // env var is the CARLOS placement code ("ie", "fleet/<name>"), a DIFFERENT |
| 226 | // vocabulary that Task 5's ChangeResourceRecordSets calls would reject |
| 227 | // batch-wide if it ever reached here (plan review catch, 2026-08-23). |
| 228 | // edgesFileRegion resolves the real thing. |
| 229 | type edgesFile struct { |
| 230 | V int `json:"v"` |
| 231 | Box string `json:"box"` |
| 232 | EIP string `json:"eip"` |
| 233 | Region string `json:"region,omitempty"` |
| 234 | // PrivateIP is the address an edge in this box's region dials (region |
| 235 | // HA spec 2 §6), and PrivateCIDR the region network that holds it, |
| 236 | // both validated by edgesfile.PrivateAddress before they are written. |
| 237 | // Routing data only. Additive and omitempty: a reader that does not know |
| 238 | // them, or finds them absent, dials EIP, as before. |
| 239 | PrivateIP string `json:"private_ip,omitempty"` |
| 240 | PrivateCIDR string `json:"private_cidr,omitempty"` |
| 241 | // Placement is this box's CARLOS placement code — CARLOS_REGION verbatim |
| 242 | // ("ie", "uk"), the OTHER region vocabulary (see Region above, issue |
| 243 | // #329). Additive and omitempty like Region: a pre-placement body parses |
| 244 | // unchanged, and a box with no CARLOS_REGION set (single-box |
| 245 | // deployments, the legal unset state) publishes no key at all. The |
| 246 | // console's create door folds every file's Placement into the armed set |
| 247 | // a requested instance region is validated against — dormant until at |
| 248 | // least one box publishes the field, so roll order does not matter. |
| 249 | Placement string `json:"placement,omitempty"` |
| 250 | // DataBucket/DataBucketRegion are this box's hibernate/data bucket |
| 251 | // coordinates (CARLOS_HIBERNATE_BUCKET/_REGION verbatim) — published so |
| 252 | // a move SOURCE can aim its staging mirror at the target box's bucket |
| 253 | // from the same file set everything else reads (instance-move spec). |
| 254 | // Additive + omitempty like Placement; a box with no hibernate bucket |
| 255 | // (activator disabled) publishes neither key. |
| 256 | DataBucket string `json:"data_bucket,omitempty"` |
| 257 | DataBucketRegion string `json:"data_bucket_region,omitempty"` |
| 258 | // HibernationConfigured is this box's own answer to "is my hibernation |
| 259 | // activator wired" — true exactly when CARLOS_HIBERNATE_BUCKET is set, |
| 260 | // which is the same env, read in the same process, that decides whether |
| 261 | // activator.ConfigFromEnv builds a Config at all (internal/host/agent.go |
| 262 | // arms the activator iff that Config is non-nil, and log.Fatalfs rather |
| 263 | // than continuing if the env is set but malformed — so a live agent |
| 264 | // process cannot publish this field disagreeing with its own activator). |
| 265 | // |
| 266 | // Deliberately NOT omitempty, and this is the whole point of the field. |
| 267 | // A reader needs to tell three states apart, and only an explicitly |
| 268 | // published `false` can carry the third: |
| 269 | // |
| 270 | // absent → the writer predates this field; its capability is UNKNOWN |
| 271 | // true → the activator is configured |
| 272 | // false → the activator is not configured |
| 273 | // |
| 274 | // Before it, the only signal was data_bucket, and its absence was |
| 275 | // ambiguous in exactly the way that matters: an old writer and a |
| 276 | // bucket-less new box produce byte-identical files. A console reading |
| 277 | // that had to guess, and the guess it made was to infer one box's |
| 278 | // silence from ANOTHER box's speech — so the first box in a fleet to |
| 279 | // publish a bucket withdrew every region that had not yet rolled, all |
| 280 | // of them healthy. One writer's upgrade cannot establish another |
| 281 | // writer's capabilities; this field is how each box answers for itself. |
| 282 | // |
| 283 | // It reports CONFIGURATION, not verified hosting readiness: a box whose |
| 284 | // IAM role cannot reach the bucket it names, or whose |
| 285 | // CARLOS_HIBERNATE_REGION is wrong, publishes true here and still fails |
| 286 | // every wake. Naming it for what it actually knows is the point. |
| 287 | HibernationConfigured bool `json:"hibernation_configured"` |
| 288 | |
| 289 | // ServesSharedPool and AcceptsInbound are this box's own answers to |
| 290 | // "would my reconcile pass mint an ORDINARY instance record" — one in |
| 291 | // the shared "" pool, needing inbound service (nearest-region spec |
| 292 | // §6.1). Deliberately NOT omitempty, HibernationConfigured's precedent: |
| 293 | // absent is a writer that predates the field (unknown), false is a box |
| 294 | // that said no. The console needs both positively true before it makes |
| 295 | // an AUTOMATIC choice of this box's region; explicit requests ignore |
| 296 | // them. Computed from the same ReconcileConfig the reconcile pass uses |
| 297 | // (edgesFileEligibility), never a second reading of the environment. |
| 298 | ServesSharedPool bool `json:"serves_shared_pool"` |
| 299 | AcceptsInbound bool `json:"accepts_inbound"` |
| 300 | |
| 301 | // SecretsKeyFP is the fingerprint of this box's CARLOS_SECRETS_KEY's |
| 302 | // public half (appconfig.KeyFingerprint), NEVER the key. Additive and |
| 303 | // omitempty, the move-any-app spec's shape: absent means this box holds |
| 304 | // no key, or one that does not parse — so it renders no exec config, |
| 305 | // and an instance placed here would run with none (nearest-region spec |
| 306 | // R9). The console requires it to equal the fingerprint of its own |
| 307 | // CARLOS_SECRETS_PUBKEY before it places anything here automatically. |
| 308 | SecretsKeyFP string `json:"secrets_key_fp,omitempty"` |
| 309 | // BoxID is the name claims use for this box (region HA spec 1 §2). |
| 310 | // Today it is the box's label: every box predates region HA and keeps |
| 311 | // it. Plan 1b adds instance ids for Auto Scaling boxes, from a positive |
| 312 | // signal rather than from being on EC2. It is NOT evidence of identity: |
| 313 | // that is the box record (internal/boxid). A reader compares the two and |
| 314 | // treats a mismatch as an ineligible box. |
| 315 | BoxID string `json:"box_id,omitempty"` |
| 316 | // Capabilities gates multi-box regions (spec §8): "region-ha-1" once |
| 317 | // CARLOS_REGION_HA is on. |
| 318 | Capabilities []string `json:"capabilities,omitempty"` |
| 319 | // Draining is set while the placer empties this box (spec 4). |
| 320 | Draining bool `json:"draining,omitempty"` |
| 321 | // Headroom is what placement chooses by (spec 2). It is refreshed when |
| 322 | // the file is written for another reason and on the loop's own tick; a |
| 323 | // headroom change alone never forces a write (see edgesFileWrite). |
| 324 | Headroom *edgeHeadroom `json:"headroom,omitempty"` |
| 325 | |
| 326 | // The moves protocol floor (moves spec §11; withMovesFields). The move |
| 327 | // door admits a move only when every file of both regions shows the |
| 328 | // published floor armed here and carlos.prev judged against it. |
| 329 | // |
| 330 | // MovesProtocol is the protocol this build speaks. NOT omitempty: absent |
| 331 | // is a writer that predates moves, which the door reads as below any |
| 332 | // floor. |
| 333 | MovesProtocol int `json:"moves_protocol"` |
| 334 | // MovesFloorMinRelease is the min_release of the floor armed on this box |
| 335 | // (<DataDir>/.moves/floor.min-release); absent: none armed. |
| 336 | MovesFloorMinRelease string `json:"moves_floor_min_release,omitempty"` |
| 337 | MovesFloorState string `json:"moves_floor_state,omitempty"` |
| 338 | MovesFloorGeneration uint64 `json:"moves_floor_generation,omitempty"` |
| 339 | // PrevRelease is carlos.prev's release (carlos.prev.version); "none" |
| 340 | // when there is no carlos.prev; absent when it is unknown. |
| 341 | PrevRelease string `json:"prev_release,omitempty"` |
| 342 | // PrevAllowedFor is the verdict (carlos.prev.allowed): the min_release |
| 343 | // carlos.prev was judged at or above. Absent: no verdict. |
| 344 | PrevAllowedFor string `json:"prev_allowed_for,omitempty"` |
| 345 | |
| 346 | // The seeds protocol (seeded-instances spec §3.3), read by the console's |
| 347 | // seedRegionReady. SeedsProtocol is not omitempty: 0 is a real answer |
| 348 | // (a build that cannot place a seed). SeedsFloorMinRelease is the |
| 349 | // armed seeds floor's min_release (absent: not armed), and |
| 350 | // PrevSeedsAllowedFor the floor carlos.prev was judged at or above. |
| 351 | SeedsFloorState string `json:"seeds_floor_state,omitempty"` |
| 352 | SeedsFloorGeneration uint64 `json:"seeds_floor_generation,omitempty"` |
| 353 | SeedsProtocol int `json:"seeds_protocol"` |
| 354 | SeedsFloorMinRelease string `json:"seeds_floor_min_release,omitempty"` |
| 355 | PrevSeedsAllowedFor string `json:"prev_seeds_allowed_for,omitempty"` |
| 356 | |
| 357 | Generation string `json:"generation"` |
| 358 | Hosts map[string]edgeHost `json:"hosts"` |
| 359 | } |
| 360 | |
| 361 | // edgesFilePass builds this box's edges file from the routes currently in |
| 362 | // reg (every SERVED row on this box, every SERVING kind — the split between |
| 363 | // those belongs to the reader, Task 3, not to this writer; the split between |
| 364 | // serving and NOT, which no reader of this file can make, belongs here — see |
| 365 | // the sidecar guard in the loop) and publishes it at edgesFileKey(box) when |
| 366 | // it changed. |
| 367 | // |
| 368 | // The change gate is the marshaled body WITH Generation cleared: Generation |
| 369 | // is a fresh RFC3339 stamp every pass, so comparing the full body would |
| 370 | // write on every tick even when the box's homed hosts and EIP are |
| 371 | // identical — a timestamp-only refresh is not a change worth a bucket |
| 372 | // write. *last holds the previous pass's gate body (also Generation- |
| 373 | // cleared) and is updated only after a successful write, mirroring |
| 374 | // registrySnapshotPass's *lastFP contract. |
| 375 | // |
| 376 | // eip may be "" (readPublicIP's "unknown" case, e.g. not on EC2, IMDS |
| 377 | // disabled, or a transient metadata-endpoint failure). A blank reading is |
| 378 | // NOT taken at face value: this box's public IP does not change tick to |
| 379 | // tick, so "" almost always means the probe hiccuped, not that the address |
| 380 | // vanished — publishing it verbatim would blank a known-good EIP for |
| 381 | // every remote edge's dial address for up to one loop cadence. When eip is |
| 382 | // "" and *last already holds a published file, this pass reuses that |
| 383 | // file's EIP instead. "" only ever reaches the wire for a box that has |
| 384 | // NEVER had a discoverable IP — the DR runbook's hosts list still gets |
| 385 | // published in that case, just without a dial address. |
| 386 | // |
| 387 | // region is the AWS region resolved once at loop arm by edgesFileRegion |
| 388 | // (the domains loop's read-once precedent — see edgesFileLoop) and is part |
| 389 | // of the change gate like every other field except Generation: a region |
| 390 | // relabel is a real change and must write, not be swallowed as a |
| 391 | // timestamp-only refresh would be. |
| 392 | // withheldAliasSaid throttles the not-published line in edgesFilePass to |
| 393 | // once per alias per process (cleared when the alias publishes). |
| 394 | var withheldAliasSaid sync.Map |
| 395 | |
| 396 | // edgesFilePass publishes one file. ident is this box's IDENTITY half of |
| 397 | // the file — Box, Region, Placement, DataBucket, DataBucketRegion — built |
| 398 | // once by the loop; eip is passed per pass because it is re-probed every |
| 399 | // tick and held across blanks (below). Hosts are derived here. |
| 400 | func edgesFilePass(ctx context.Context, s release.Store, reg *registry.Registry, ident edgesFile, eip string, last *[]byte) error { |
| 401 | return edgesFileWrite(ctx, s, reg, ident, eip, last, false) |
| 402 | } |
| 403 | |
| 404 | // edgesFileWrite is edgesFilePass with one more input. refresh is true on |
| 405 | // the loop's own timed pass: the file is then written even if nothing but |
| 406 | // its headroom reading changed, so headroom is never more than one timer |
| 407 | // period stale. On every other pass a headroom change alone is not a |
| 408 | // reason to write. |
| 409 | func edgesFileWrite(ctx context.Context, s release.Store, reg *registry.Registry, ident edgesFile, eip string, last *[]byte, refresh bool) error { |
| 410 | return edgesFileWriteClaims(ctx, s, reg, ident, eip, last, refresh, nil) |
| 411 | } |
| 412 | |
| 413 | // errBoxAddressesUnknown fails an edges file pass whose address source could |
| 414 | // not tell what this box holds (region HA on). Nothing has been written. |
| 415 | var errBoxAddressesUnknown = errors.New("edges file: the box's addresses could not be read") |
| 416 | |
| 417 | // edgesFileWriteClaims is edgesFileWrite with the claim reader region HA |
| 418 | // adds (edgesclaims.go). claims is nil unless CARLOS_REGION_HA is on, and |
| 419 | // nil publishes exactly what edgesFileWrite always has. |
| 420 | func edgesFileWriteClaims(ctx context.Context, s release.Store, reg *registry.Registry, ident edgesFile, eip string, last *[]byte, refresh bool, claims *edgesClaims) error { |
| 421 | box := ident.Box |
| 422 | routes, err := reg.List() |
| 423 | if err != nil { |
| 424 | return fmt.Errorf("edges: list: %w", err) |
| 425 | } |
| 426 | hosts := make(map[string]edgeHost, len(routes)) |
| 427 | for _, r := range routes { |
| 428 | // edge.Reload (internal/edge/edge.go) is the deployment's own |
| 429 | // definition of "serves": it skips any row whose Status != "ready" |
| 430 | // because a pending/creating/failed row has no working backend and |
| 431 | // is never dialed by any edge, home or remote. Mirroring that gate |
| 432 | // here means this file never claims role:"home" for a host nothing |
| 433 | // actually serves. Hibernating routes stay Status "ready" (that is |
| 434 | // what makes wake-on-request possible), so nothing reachable is |
| 435 | // excluded by this filter. |
| 436 | if r.Status != "ready" { |
| 437 | continue |
| 438 | } |
| 439 | // A sidecar is NEVER advertised as served — by this box or any |
| 440 | // other. This is publishResidentFleetBindings' guard on the file |
| 441 | // that superseded it, and it is load-bearing rather than tidy, |
| 442 | // because the edge's own Kind refusal cannot reach this path: that |
| 443 | // refusal lives in Reload, which builds THIS box's routing map from |
| 444 | // rows it holds. A remote edge never sees the row at all. It sees |
| 445 | // this file, misses on its own upstream lookup, and takes the |
| 446 | // backhaul arm (edge.go's e.backhaul → homeMap) — dialing the home |
| 447 | // box with the sidecar's name as SNI, which is a live proxy to the |
| 448 | // one member the platform guarantees nothing can dial. The same |
| 449 | // entry also passes deploymentHost, so the name becomes orderable |
| 450 | // by the cert machinery deployment-wide. |
| 451 | // |
| 452 | // So the writer's "ALL kinds, the serving-class split belongs to |
| 453 | // the reader" rule above holds only while every kind is servable. |
| 454 | // It stops holding at the first row that is unroutable BY DESIGN, |
| 455 | // and a reader that cannot tell (a remote edge, reading JSON) is |
| 456 | // not a reader that can make the split. |
| 457 | if r.IsSidecar() { |
| 458 | continue |
| 459 | } |
| 460 | // Keys are registry row Host values verbatim, INCLUDING *.parent |
| 461 | // catch-all wildcard rows (Task 3's reconcile pass derives these |
| 462 | // from a claim's catch-all row, same as edge.go's upstream map) — |
| 463 | // this writer performs no expansion. A reader that needs exact-host |
| 464 | // answers must apply edge.go's lookupUpstream one-label wildcard |
| 465 | // rule itself: an exact match wins, and on a miss "*.<parent>" |
| 466 | // (the leading dot of the first label only) is tried next. |
| 467 | // |
| 468 | // The replica mark rides the ROW (#162): a route minted from a |
| 469 | // role=replica record publishes with replica:true wherever its own |
| 470 | // host appears, so every reader of the file can tell the account's |
| 471 | // homes from its checkout replicas. See the replica loop below for |
| 472 | // the account apex that rides along. |
| 473 | hosts[r.Host] = edgeHost{Role: "home", Platform: r.Platform || r.Provisioned, Replica: r.Replica} |
| 474 | } |
| 475 | // Region HA (spec 1 §5.4): a relocatable host is home only on the box |
| 476 | // its claim names, stamped with the installation. Before owners and |
| 477 | // aliases, so an alias of a withheld host is withheld with it. |
| 478 | applyEdgesClaims(ctx, claims, routes, hosts) |
| 479 | // Owners (#175 P7): computed over the served rows alone, before any alias |
| 480 | // or catch-all joins the map, so an alias can only ever inherit a route's. |
| 481 | paths, err := reg.ListPaths() |
| 482 | if err != nil { |
| 483 | return fmt.Errorf("edges: list paths: %w", err) |
| 484 | } |
| 485 | owners := servedOwners(routes, hosts, paths) |
| 486 | for host, h := range hosts { |
| 487 | h.Account, h.App = owners[host].owner() |
| 488 | hosts[host] = h |
| 489 | } |
| 490 | // Attached custom domains (#151) live in route_aliases, not routes, and |
| 491 | // were absent from this file (#287): a non-home edge neither backhauled |
| 492 | // them (Home missed) nor answered their validations or let the issuer |
| 493 | // order for them (KnownHost missed). Publish each alias whose primary is |
| 494 | // a served row above, under the alias's own name — the reader wants |
| 495 | // exact names, and every reader already does the right thing with one: |
| 496 | // Home dials the home box with the alias as SNI, and the home box's own |
| 497 | // upstream map serves it. Bounds, each doing work: |
| 498 | // |
| 499 | // - only an alias whose DNS the sweep has stamped "pointing" (at one of |
| 500 | // PLATFORM_EDGE_IPS). Attach needs no DNS proof, so the alias set is |
| 501 | // tenant-chosen and unbounded; publishing every one would hand a |
| 502 | // tenant an unbounded set of names the shared issuer will order for |
| 503 | // (#277's amplification envelope covers routes, which the platform |
| 504 | // provisions). A name that actually resolves to this deployment is |
| 505 | // the name whose validations must be answerable everywhere; |
| 506 | // - never platform-class: a customer's name derives no wildcard parent; |
| 507 | // - marked alias: the reader lets a real route on another box beat it; |
| 508 | // - no "*.parent" catch-all alias: a remote edge could dial it but has |
| 509 | // no cert for it (delegated wildcard parents are box-local), so it |
| 510 | // stays exactly as before — served on its home box only; |
| 511 | // - an alias shadowed by a real route here is the route's, as |
| 512 | // edge.Reload rules. |
| 513 | aliases, err := reg.ListAliases() |
| 514 | if err != nil { |
| 515 | return fmt.Errorf("edges: list aliases: %w", err) |
| 516 | } |
| 517 | for _, a := range aliases { |
| 518 | if a.Alias == "" || strings.HasPrefix(a.Alias, "*.") { |
| 519 | continue |
| 520 | } |
| 521 | if _, served := hosts[a.Host]; !served { |
| 522 | continue |
| 523 | } |
| 524 | if a.DNSStatus != "pointing" { |
| 525 | // Withheld, and said so ONCE per name per process: an alias |
| 526 | // whose DNS resolves to an address outside PLATFORM_EDGE_IPS |
| 527 | // (a region EIP missing from that list is the usual cause) is |
| 528 | // otherwise served on its home box only, silently. |
| 529 | if _, said := withheldAliasSaid.LoadOrStore(a.Alias, true); !said { |
| 530 | log.Printf("edges: alias %s not published — dns_status %q, want \"pointing\" (does PLATFORM_EDGE_IPS list every armed edge?)", a.Alias, a.DNSStatus) |
| 531 | } |
| 532 | continue |
| 533 | } |
| 534 | withheldAliasSaid.Delete(a.Alias) |
| 535 | if _, taken := hosts[a.Alias]; taken { |
| 536 | continue |
| 537 | } |
| 538 | // The edge serves an alias with its primary's own entry, prefix map |
| 539 | // included, so the alias's owner is the primary's, never the alias |
| 540 | // row's own columns. |
| 541 | acct, app := owners[a.Host].owner() |
| 542 | // The alias is served by its primary's installation, so it carries |
| 543 | // the primary's stamp (region HA HA4): an unstamped alias would tie |
| 544 | // with a dead predecessor's and the edge would keep the first box |
| 545 | // by name. |
| 546 | hosts[a.Alias] = edgeHost{Role: "home", Alias: true, Account: acct, App: app, Stamp: hosts[a.Host].Stamp} |
| 547 | } |
| 548 | |
| 549 | // Replica placements (#162): a row minted from a role=replica record — |
| 550 | // this box holds it BECAUSE its placement is this box's (the reconciler's |
| 551 | // scope test is the only mint gate) — publishes with the replica mark |
| 552 | // set, which the main loop above already stamps onto its own host (the |
| 553 | // mark rides the ROW). Nothing beyond that host is claimed: the home's |
| 554 | // hostname and the platform apex live on the home box's rows, and a name |
| 555 | // this box cannot serve from a local row must not be claimed (a |
| 556 | // replica-claimed apex would let foldHomes' first-sorted-box rule hand |
| 557 | // the apex DIAL target to a replica box fleet-wide). Custom domains |
| 558 | // reach the edges by their owner's CNAME to edge.<apps-domain> anyway |
| 559 | // (the steering fold's own documented path). |
| 560 | |
| 561 | // Delegated customer wildcards (#305): the wildcard_domains rows are the |
| 562 | // parents this box obtains "*.<parent>" for (a claim with the Wildcard |
| 563 | // capability, primary homed here). Published as the catch-all key with |
| 564 | // its own class so a remote edge both dials one-label hosts under it |
| 565 | // (Home's rule) and loads the parent's mirrored wildcard PEM — the two |
| 566 | // halves #302 could not deliver from an alias row alone. Never platform. |
| 567 | wds, err := reg.ListWildcardDomains() |
| 568 | if err != nil { |
| 569 | return fmt.Errorf("edges: list wildcard domains: %w", err) |
| 570 | } |
| 571 | // Two halves, published separately: the CERT half (every edge loads the |
| 572 | // parent's PEM) always; the DIAL half (role home — remote edges backhaul |
| 573 | // one-label hosts under the parent) only when this box actually serves |
| 574 | // the catch-all, i.e. the claim has Catchall and its "*.<parent>" alias |
| 575 | // row points at a served primary. A Wildcard claim without Catchall |
| 576 | // serves only its exact aliases (published above, each on its own |
| 577 | // terms); advertising a dial target for it would have region edges |
| 578 | // terminate and backhaul any invented name under the parent to a box |
| 579 | // that answers 503. |
| 580 | // catchallServed maps each served parent to the primary its "*.<parent>" |
| 581 | // alias row points at, the route the edge serves the catch-all with. |
| 582 | catchallServed := map[string]string{} |
| 583 | for _, a := range aliases { |
| 584 | if parent, ok := strings.CutPrefix(a.Alias, "*."); ok { |
| 585 | if _, served := owners[a.Host]; served { |
| 586 | catchallServed[parent] = a.Host |
| 587 | } |
| 588 | } |
| 589 | } |
| 590 | for _, wd := range wds { |
| 591 | if wd.Host == "" || strings.HasPrefix(wd.Host, "*.") { |
| 592 | continue |
| 593 | } |
| 594 | key := "*." + wd.Host |
| 595 | if prev, taken := hosts[key]; taken { |
| 596 | // A real route owns the key (its catch-all row, published above): |
| 597 | // the route keeps its role and class, and the parent's class is |
| 598 | // ADDED — the reader's fold merges classes the same way, and a |
| 599 | // parent whose PEM must load everywhere is a fact about the name, |
| 600 | // not about which row got there first. |
| 601 | prev.Wildcard = true |
| 602 | hosts[key] = prev |
| 603 | continue |
| 604 | } |
| 605 | // Marked alias whenever the DIAL half is on, and the mark is |
| 606 | // load-bearing rather than cosmetic. The catch-all key never reaches |
| 607 | // the alias loop above (it skips "*." rows on purpose), so this is |
| 608 | // the only place its class can be set — and edgesfile.DeploymentRoutes |
| 609 | // reads exactly `Role == RoleHome && !Alias` as "some live box serves |
| 610 | // this hostname as a real route". Published unmarked, a served |
| 611 | // catch-all is therefore evidence against itself: the next |
| 612 | // domains.ReconcileAliases pass reads our own file back, refuses the |
| 613 | // claim that produced the row, and drops the alias — which turns the |
| 614 | // dial half off, unmarks nothing, and lets the pass after that |
| 615 | // recreate it. That loop is not theoretical; it ran on *.woodstar.app |
| 616 | // every 5-6 minutes, taking the name down about half of every hour. |
| 617 | // A catch-all we serve IS a materialized claim, which is what the |
| 618 | // mark means, so say so. |
| 619 | role, alias := "wildcard", false |
| 620 | var acct, app string |
| 621 | var stamp int64 |
| 622 | if primary, served := catchallServed[wd.Host]; served { |
| 623 | role, alias = "home", true |
| 624 | stamp = hosts[primary].Stamp // the dial half is the primary's installation |
| 625 | // The owner is the wildcard claim's, and only when the primary |
| 626 | // the catch-all is served by (with its prefixes) agrees with it: |
| 627 | // every route behind the name has to have one owner. |
| 628 | o := *owners[primary] |
| 629 | o.add(wd.Account, wd.App) |
| 630 | acct, app = o.owner() |
| 631 | } |
| 632 | hosts[key] = edgeHost{Role: role, Wildcard: true, Alias: alias, Account: acct, App: app, Stamp: stamp} |
| 633 | } |
| 634 | |
| 635 | var private BoxAddresses |
| 636 | if claims != nil && claims.addrs != nil { |
| 637 | // Region HA spec 2 §6 (A1-12): publish only what the box's address |
| 638 | // source reports now. An authoritative blank publishes no address, |
| 639 | // never the last one: that may be the region's EIP, which the load |
| 640 | // balancer has since taken. An unknown read (a metadata hiccup) |
| 641 | // writes nothing: the previous file stands and the loop retries, |
| 642 | // so one failed request does not hide a healthy box from its |
| 643 | // readers. |
| 644 | read, ok := claims.addrs.BoxAddresses() |
| 645 | if !ok { |
| 646 | return errBoxAddressesUnknown |
| 647 | } |
| 648 | private = read |
| 649 | eip = private.Public |
| 650 | } else if eip == "" && *last != nil { |
| 651 | var prior edgesFile |
| 652 | if err := json.Unmarshal(*last, &prior); err == nil { |
| 653 | eip = prior.EIP |
| 654 | } |
| 655 | } |
| 656 | |
| 657 | f := ident |
| 658 | f.V, f.EIP, f.Hosts, f.Generation = 1, eip, hosts, "" |
| 659 | f.PrivateIP, f.PrivateCIDR = private.Private, private.PrivateCIDR |
| 660 | gateCopy := f |
| 661 | gateCopy.Headroom = nil // a changing reading is not, by itself, a reason to write |
| 662 | gate, err := json.Marshal(gateCopy) |
| 663 | if err != nil { |
| 664 | return fmt.Errorf("edges: marshal: %w", err) |
| 665 | } |
| 666 | if bytes.Equal(gate, *last) && !(refresh && f.Headroom != nil) { |
| 667 | return nil |
| 668 | } |
| 669 | |
| 670 | f.Generation = time.Now().UTC().Format(time.RFC3339) |
| 671 | body, err := json.Marshal(f) |
| 672 | if err != nil { |
| 673 | return fmt.Errorf("edges: marshal: %w", err) |
| 674 | } |
| 675 | if err := putReplacing(ctx, s, edgesFileKey(box), body); err != nil { |
| 676 | return err |
| 677 | } |
| 678 | *last = gate |
| 679 | return nil |
| 680 | } |
| 681 | |
| 682 | // seedEdgesFileGate reads this box's last-published edges file at loop arm |
| 683 | // and returns the change-gate body (Generation cleared) to prime *last, |
| 684 | // mirroring seedSnapshotFingerprint's contract exactly (snapshot_loop.go): |
| 685 | // without this, a freshly started (or crash-looping) process's in-memory |
| 686 | // *last starts empty, so edgesFilePass's own change gate — bytes.Equal(gate, |
| 687 | // *last) — can never match on the first tick and the process republishes an |
| 688 | // UNCHANGED file on every restart, burning an ETag/ifMatch round trip for |
| 689 | // nothing (plan-2 final review, Important 1, first half). |
| 690 | // |
| 691 | // The same read also recovers the EIP hold for free: edgesFilePass's blank- |
| 692 | // probe fallback reads *last's EIP when this tick's probe is "", and the |
| 693 | // seeded gate still carries the prior file's EIP (only Generation is |
| 694 | // cleared). Without this seed, a box that restarts and hits a transient |
| 695 | // IMDS blip on its very first tick has a nil *last, the hold's `*last != |
| 696 | // nil` guard never fires, and the box would publish "" — blanking a known- |
| 697 | // good EIP for every remote edge's dial address until the next successful |
| 698 | // probe (the same review's Important 1, second half; the restart-blip |
| 699 | // case). One seed read closes both. |
| 700 | // |
| 701 | // Any read or parse failure — no prior file yet, a corrupt object, a store |
| 702 | // that is down — returns nil: the loop then behaves exactly as it always |
| 703 | // has on a box's first-ever tick. This must never fail the loop. |
| 704 | func seedEdgesFileGate(ctx context.Context, s release.Store, box string) []byte { |
| 705 | body, _, err := s.Get(ctx, edgesFileKey(box)) |
| 706 | if err != nil { |
| 707 | return nil |
| 708 | } |
| 709 | var prior edgesFile |
| 710 | if err := json.Unmarshal(body, &prior); err != nil { |
| 711 | return nil |
| 712 | } |
| 713 | prior.Generation = "" |
| 714 | prior.Headroom = nil // the gate never includes headroom: see edgesFileWrite |
| 715 | gate, err := json.Marshal(prior) |
| 716 | if err != nil { |
| 717 | return nil |
| 718 | } |
| 719 | return gate |
| 720 | } |
| 721 | |
| 722 | // edgesFileRegion resolves this box's AWS region, pure and getenv/probe- |
| 723 | // injectable — domainsTickConfig's own shape (agent.go): the network probe |
| 724 | // (readPlacementRegion, an IMDS call) is read ONCE by the caller and handed |
| 725 | // in as probedRegion, so this function itself never touches the network and |
| 726 | // its override/unset/trim contract is unit-pinned without a live IMDS |
| 727 | // server in the test. |
| 728 | // |
| 729 | // CARLOS_AWS_REGION, when set, wins outright — the escape hatch for testing |
| 730 | // and for any box this build ever runs on OFF EC2. Otherwise the IMDS |
| 731 | // probe's answer stands, trimmed. Neither set means "" — a box that is |
| 732 | // off EC2 with no override is correctly UNARMED for steering (Task 4's |
| 733 | // converger gates on this field parsing as an AWS region, never just |
| 734 | // non-empty), not an error: DNS health checks target EIPs, which never |
| 735 | // depended on this. |
| 736 | func edgesFileRegion(getenv func(string) string, probedRegion string) string { |
| 737 | if override := strings.TrimSpace(getenv("CARLOS_AWS_REGION")); override != "" { |
| 738 | return override |
| 739 | } |
| 740 | return strings.TrimSpace(probedRegion) |
| 741 | } |
| 742 | |
| 743 | // edgesFileEligibility derives the two published eligibility answers from |
| 744 | // the reconcile config this box actually runs with — provisionTickConfig, |
| 745 | // the same parse the provision tick uses — so a spelling the reconciler |
| 746 | // reads one way can never be published another. |
| 747 | func edgesFileEligibility(getenv func(string) string) (sharedPool, inbound bool) { |
| 748 | return provisionTickConfig(getenv).SharedInstanceEligibility() |
| 749 | } |
| 750 | |
| 751 | // edgesFileSecretsFP fingerprints the box's secrets key, read the same way |
| 752 | // the config loops read it (agent.go configSyncLoop). The key itself never |
| 753 | // leaves this function. |
| 754 | func edgesFileSecretsFP(getenv func(string) string) string { |
| 755 | raw := strings.TrimSpace(getenv("CARLOS_SECRETS_KEY")) |
| 756 | if raw == "" { |
| 757 | return "" |
| 758 | } |
| 759 | priv, err := appconfig.ParsePrivateKey(raw, "CARLOS_SECRETS_KEY") |
| 760 | if err != nil { |
| 761 | log.Printf("agent: CARLOS_SECRETS_KEY does not parse, so no secrets_key_fp is published: %v", err) |
| 762 | return "" |
| 763 | } |
| 764 | return appconfig.KeyFingerprint(priv.PublicKey()) |
| 765 | } |
| 766 | |
| 767 | // edgesFileLoop publishes this box's edges file every 5 minutes + jitter — |
| 768 | // the same cadence and arming site as registrySnapshotLoop — AND within one |
| 769 | // watch interval of any change to this box's registry (#174). |
| 770 | // |
| 771 | // The change path is what a new host needs. Every other edge learns that this |
| 772 | // box homes a host from this file (internal/edge's homeMap mirror, a 30s |
| 773 | // poll), and until it has, a visitor steered to one of them gets that edge's |
| 774 | // unknown-host answer. On the timer alone that window was the whole 5min + |
| 775 | // jitter cycle plus the poll: a titogo sandbox host served here at +51s was a |
| 776 | // dead end on the uk edge for about a minute more, and could have been for |
| 777 | // six. Watching the registry fingerprint (the same 2s poll edge.Watch |
| 778 | // reloads on) closes the writer's half to seconds; what is left is the |
| 779 | // reader's 30s poll. |
| 780 | // |
| 781 | // It stays cheap because edgesFilePass is: local registry reads, and a |
| 782 | // bucket write only when the file's body (Generation aside) changed. A |
| 783 | // deploy or a health flip moves the fingerprint without moving the hosts, |
| 784 | // so it costs a pass and no write. |
| 785 | // |
| 786 | // The trigger covers what a new host needs, not every input to the file. |
| 787 | // Fingerprint hashes a route's row and status (so a route turning ready |
| 788 | // moves it), aliases' names and wildcard_domains; it does not hash a route's |
| 789 | // platform, provisioned or kind columns, nor an alias's dns_status. A change |
| 790 | // to only those still waits for the timer, and so does a newly attached |
| 791 | // custom domain: its pass runs at once but withholds it until the DNS sweep |
| 792 | // stamps it "pointing". |
| 793 | // |
| 794 | // The timer is kept for what the fingerprint deliberately does not see: |
| 795 | // route_aliases' dns_status (a cadence-stamped column Fingerprint excludes) |
| 796 | // decides which aliases are published, and a failed write must be retried |
| 797 | // even if nothing else changes. |
| 798 | // |
| 799 | // Platform-box job, retentionLoop's rules: skipped on fleet boxes, disabled |
| 800 | // with a log line when there is no deployment bucket. Single-box deployments: |
| 801 | // the file is published and nothing reads it yet. |
| 802 | func edgesFileLoop(reg *registry.Registry) { |
| 803 | ctx := context.Background() |
| 804 | if shouldSkipPublisherElection(os.Getenv) { |
| 805 | log.Printf("agent: edges file loop is a platform-box job — skipped on a fleet box") |
| 806 | return |
| 807 | } |
| 808 | store, err := openDeployStore(ctx) |
| 809 | if err != nil { |
| 810 | log.Printf("agent: edges file loop disabled: %v", err) |
| 811 | return |
| 812 | } |
| 813 | box := metricsBoxLabel(os.Getenv, readMachineID()) |
| 814 | // Eligibility is read once here at arm, from the reconcile config's own |
| 815 | // parse, never a second reading of CARLOS_POOLS or CARLOS_INBOUND. |
| 816 | sharedPool, inbound := edgesFileEligibility(os.Getenv) |
| 817 | // Region resolved once here at arm — the domains loop's read-once |
| 818 | // precedent (domainsLoop reads its env once before the loop and threads |
| 819 | // the result through every tick via cfg, never re-reading or re-probing |
| 820 | // per pass). readPlacementRegion is the one IMDS call; edgesFileRegion |
| 821 | // applies the CARLOS_AWS_REGION override on top, pure. |
| 822 | region := edgesFileRegion(os.Getenv, readPlacementRegion()) |
| 823 | // The placement code alongside it: CARLOS_REGION, the same env the |
| 824 | // reconcile pass scopes by — the two vocabularies travel in one file so |
| 825 | // a reader can join them without a second source. |
| 826 | ident := edgesFile{ |
| 827 | Box: box, |
| 828 | Region: region, |
| 829 | Placement: strings.TrimSpace(os.Getenv("CARLOS_REGION")), |
| 830 | DataBucket: strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_BUCKET")), |
| 831 | DataBucketRegion: strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_REGION")), |
| 832 | // Same env, same trim, one line apart, deliberately: this boolean |
| 833 | // must never disagree with the bucket beside it, and the way to |
| 834 | // guarantee that is to derive both from the one read rather than to |
| 835 | // ask two questions that could drift. |
| 836 | HibernationConfigured: strings.TrimSpace(os.Getenv("CARLOS_HIBERNATE_BUCKET")) != "", |
| 837 | ServesSharedPool: sharedPool, |
| 838 | AcceptsInbound: inbound, |
| 839 | SecretsKeyFP: edgesFileSecretsFP(os.Getenv), |
| 840 | BoxID: box, |
| 841 | Capabilities: edgesFileCapabilities(os.Getenv), |
| 842 | } |
| 843 | // The flag-derived capabilities; offline-move-1 joins them on each pass |
| 844 | // once the moves worker is armed and capable, and db-read-1 joins them |
| 845 | // on each pass while the armed db-reads loop's last readiness probe |
| 846 | // passed (db region forward spec §7.1). The db-reads agent's draining |
| 847 | // check reads the draining flag this file publishes, stored on each |
| 848 | // pass, so the box and the console's evidence rule read one value. |
| 849 | baseCaps := ident.Capabilities |
| 850 | dataDir := provisionTickConfig(os.Getenv).DataDir |
| 851 | optDir := release.OptDir() |
| 852 | // The local floor files this file publishes are refreshed hourly by |
| 853 | // their own loop, on this loop's store handle (moves spec §11). |
| 854 | go movesFloorLoop(ctx, store, dataDir) |
| 855 | // Region HA reads each relocatable host's claim (spec 1 §5.4); off, the |
| 856 | // file is exactly what it always was. |
| 857 | var claims *edgesClaims |
| 858 | if regionHAEnabled(os.Getenv) { |
| 859 | claims = newEdgesClaims(box, func(ctx context.Context, host string) (hostclaim.Observed, error) { |
| 860 | return hostclaim.Read(ctx, store, host) |
| 861 | }) |
| 862 | } |
| 863 | log.Printf("agent: edges file loop armed (5min + jitter, and %s after a registry change, %s)", edgesFileWatchInterval, edgesFileKey(box)) |
| 864 | last := seedEdgesFileGate(ctx, store, box) |
| 865 | sched := newEdgesFileSchedule(time.Now(), func() time.Duration { |
| 866 | return 5*time.Minute + rand.N(time.Minute) |
| 867 | }) |
| 868 | state := &edgesLoopState{sched: sched, claims: claims} |
| 869 | tick := time.NewTicker(edgesFileWatchInterval) |
| 870 | defer tick.Stop() |
| 871 | fpErrs, passErrs := 0, 0 |
| 872 | // The EIP cannot change under a running box, so it is read once here and |
| 873 | // re-read only on the timer's passes, never on every change pass: off |
| 874 | // EC2 each read can block for the probe's whole timeout. |
| 875 | eip := readPublicIP() |
| 876 | for now := range tick.C { |
| 877 | fp, err := reg.Fingerprint() |
| 878 | if err != nil { |
| 879 | // Not fatal: the timer still publishes. Counted and throttled the |
| 880 | // way edge.Watch counts its own poll errors. |
| 881 | fpErrs++ |
| 882 | if fpErrs == 1 || fpErrs%300 == 0 { |
| 883 | log.Printf("agent: edges file registry poll error (#%d): %v", fpErrs, err) |
| 884 | } |
| 885 | } else { |
| 886 | fpErrs = 0 |
| 887 | } |
| 888 | if !state.due(now, fp, err == nil) { |
| 889 | continue |
| 890 | } |
| 891 | if state.timed() { |
| 892 | eip = readPublicIP() |
| 893 | } |
| 894 | ident.Capabilities = withDBReadCapability(withOfflineCapability(baseCaps, armedOfflineCapable()), dbReadsCapable()) |
| 895 | dbReadsDraining.Store(ident.Draining) |
| 896 | ident.Headroom = headroomFor(headroomCapabilities(ident.Capabilities), dataDir, reg) |
| 897 | ident = withMovesFields(ident, dataDir, optDir) |
| 898 | ident = withSeedsFields(ident, dataDir, optDir) |
| 899 | err = edgesFileWriteClaims(ctx, store, reg, ident, eip, &last, state.timed(), claims) |
| 900 | if err != nil { |
| 901 | // Throttled: a bucket that refuses every write (an IAM Deny, an |
| 902 | // outage) is retried on a backoff, and each retry must not be a |
| 903 | // journal line. |
| 904 | passErrs++ |
| 905 | if passErrs == 1 || passErrs%30 == 0 { |
| 906 | log.Printf("agent: edges file pass failed (#%d): %v", passErrs, err) |
| 907 | } |
| 908 | } else { |
| 909 | passErrs = 0 |
| 910 | } |
| 911 | state.done(now, err == nil) |
| 912 | } |
| 913 | } |
| 914 | |
| 915 | // edgesFileWatchInterval is how often edgesFileLoop polls the registry |
| 916 | // fingerprint: edge.Watch's own 2s, so this file follows a route change on |
| 917 | // the same beat the local edge starts serving it. |
| 918 | const edgesFileWatchInterval = 2 * time.Second |
| 919 | |
| 920 | // edgesFileSchedule decides, tick by tick, whether edgesFileLoop runs a |
| 921 | // pass. Pulled out of the loop so the decision is testable without a |
| 922 | // bucket, a registry or a clock. |
| 923 | // |
| 924 | // A pass is due when the timer is up, or when the fingerprint moved since |
| 925 | // the last SUCCESSFUL pass. The first fingerprint seen only baselines: the |
| 926 | // first timed pass stays a minute after arm, as it always was, rather than |
| 927 | // publishing in the loop's first two seconds. |
| 928 | // |
| 929 | // The baseline advances only when a pass succeeds, so a change whose pass |
| 930 | // failed is retried rather than lost until the timer — the rule edge.Watch |
| 931 | // keeps for a failed Reload. Unlike a Reload, a pass is a bucket write, so |
| 932 | // the retries back off: 2s doubling to the timer's own cadence, reset by the |
| 933 | // next success. A bucket that refuses every write is therefore asked about |
| 934 | // as often as it was before the change trigger existed, not every tick. |
| 935 | type edgesFileSchedule struct { |
| 936 | next time.Time // when the timer is next due |
| 937 | jitter func() time.Duration // the cadence after each timed pass |
| 938 | seen string // the fingerprint on the tick being decided |
| 939 | baseline string // the fingerprint the last good pass covered |
| 940 | armed bool // a baseline exists |
| 941 | timed bool // the pass being run is the timer's |
| 942 | retryAt time.Time // no change pass before this, after a failure |
| 943 | backoff time.Duration // the wait after the next failure |
| 944 | } |
| 945 | |
| 946 | // Bounds on the change-pass retry backoff. |
| 947 | const ( |
| 948 | edgesFileRetryMin = 2 * time.Second |
| 949 | edgesFileRetryMax = 5 * time.Minute |
| 950 | ) |
| 951 | |
| 952 | func newEdgesFileSchedule(now time.Time, jitter func() time.Duration) *edgesFileSchedule { |
| 953 | return &edgesFileSchedule{next: now.Add(time.Minute), jitter: jitter} |
| 954 | } |
| 955 | |
| 956 | // due reports whether to run a pass on this tick. fpOK=false means the |
| 957 | // fingerprint could not be read, which never triggers a change pass. |
| 958 | func (s *edgesFileSchedule) due(now time.Time, fp string, fpOK bool) bool { |
| 959 | s.timed = !now.Before(s.next) |
| 960 | s.seen = "" |
| 961 | if fpOK { |
| 962 | s.seen = fp |
| 963 | if !s.armed { |
| 964 | s.baseline, s.armed = fp, true |
| 965 | } |
| 966 | } |
| 967 | changed := fpOK && fp != s.baseline && !now.Before(s.retryAt) |
| 968 | return s.timed || changed |
| 969 | } |
| 970 | |
| 971 | // done records the outcome of the pass due just asked for. |
| 972 | func (s *edgesFileSchedule) done(now time.Time, ok bool) { |
| 973 | if s.timed { |
| 974 | // The timer moves on whatever happened: a failed timed pass is |
| 975 | // retried by the next timed one, exactly as before this schedule. |
| 976 | s.next = now.Add(s.jitter()) |
| 977 | } |
| 978 | if ok { |
| 979 | if s.seen != "" { |
| 980 | s.baseline = s.seen |
| 981 | } |
| 982 | s.retryAt, s.backoff = time.Time{}, 0 |
| 983 | return |
| 984 | } |
| 985 | if s.backoff == 0 { |
| 986 | s.backoff = edgesFileRetryMin |
| 987 | } |
| 988 | s.retryAt = now.Add(s.backoff) |
| 989 | s.backoff = min(2*s.backoff, edgesFileRetryMax) |
| 990 | } |
| 991 | |
| 992 | // aliasClaimCache is the claim fold both alias legs share (the fast-converge |
| 993 | // leg in fullPass and the five-minute instances pass). Process-lifetime and |
| 994 | // package-level for the same reason withheldAliasSaid above is: there is one |
| 995 | // agent per process, and both legs are asking about the same deployment's |
| 996 | // claims, so two caches would double the cold reads and halve the hit rate |
| 997 | // for nothing. |
| 998 | // |
| 999 | // The fast leg is why this exists at all. It runs on every serial bump — a |
| 1000 | // seconds cadence — and ListClaims is one List plus a GET per claim in the |
| 1001 | // WHOLE deployment. At the hundreds of custom domains titogo is heading for, |
| 1002 | // that is a steady drip of bucket reads on every box, sitting ahead of the |
| 1003 | // adopt leg inside a two-second convergence promise. See domains.ClaimCache. |
| 1004 | var aliasClaimCache domains.ClaimCache |
| 1005 | |
amadan