| 1 | // The CLI's instance provisioning API: the verbs that turn "place an |
| 2 | // instance" into a data write in the deployment bucket, so a signup path |
| 3 | // holding no AWS credentials and no box access can provision one in |
| 4 | // seconds. The record it writes is the whole authorization; the agent's |
| 5 | // reconcile pass (internal/provision) is what turns it into a route, on its |
| 6 | // own box only. |
| 7 | // |
| 8 | // This was the FIRST console family a non-human principal could speak to, |
| 9 | // and for a while it carried its own resolver (apiInstancePrincipal) to do |
| 10 | // it. That resolver is gone: service principals are a platform-wide idea now |
| 11 | // (svcprincipal.go, spec 2026-08-18), and its rules — the sqid-only match, |
| 12 | // the exact app binding, the `app:<sqid>/<app>` audit actor — live in the |
| 13 | // shared path (apiAccountApp) where every family gets them. What is left |
| 14 | // here is the same gate with the instance BUDGET bound to it. |
| 15 | // |
| 16 | // Two shapes of bearer reach it: a person's token, resolved through exactly |
| 17 | // the same membership-proving lookup every other CLI verb uses, or a service |
| 18 | // credential bound to one account and — for the legacy app tokens this |
| 19 | // family minted — one app, which must match the path it is used on EXACTLY. |
| 20 | // Every refusal before that proof is the family's byte-identical 404 — an |
| 21 | // unknown account, someone else's account, an unclaimed app and a service |
| 22 | // credential probing a foreign path are all one answer, so probing teaches |
| 23 | // nothing. |
| 24 | // |
| 25 | // Spec: docs/superpowers/specs/2026-08-04-instance-provisioning-design.md |
| 26 | // §7 (the verbs), §8 (the credential), §10 (the DNS seam), §11 (security). |
| 27 | package console |
| 28 | |
| 29 | import ( |
| 30 | "context" |
| 31 | "encoding/json" |
| 32 | "errors" |
| 33 | "log" |
| 34 | "net/http" |
| 35 | "os" |
| 36 | "slices" |
| 37 | "strconv" |
| 38 | "strings" |
| 39 | "time" |
| 40 | |
| 41 | "github.com/carlosframework/platform/internal/gate" |
| 42 | "github.com/carlosframework/platform/internal/hostops" |
| 43 | "github.com/carlosframework/platform/internal/provision" |
| 44 | "github.com/carlosframework/platform/internal/registry" |
| 45 | "github.com/carlosframework/platform/internal/release" |
| 46 | "github.com/carlosframework/platform/internal/status" |
| 47 | ) |
| 48 | |
| 49 | // instanceStoreTimeout bounds every store round trip these handlers make, |
| 50 | // the same 15s the rest of the CLI API uses: a wedged bucket costs one |
| 51 | // request 15 seconds, not a console goroutine forever. |
| 52 | const instanceStoreTimeout = 15 * time.Second |
| 53 | |
| 54 | // instanceListMax caps one list answer (spec §7): instance fleets can be |
| 55 | // large, and a caller that needs more than 500 is not using this door for |
| 56 | // what it is. The response says so with `truncated` rather than silently |
| 57 | // serving a prefix. |
| 58 | const instanceListMax = 500 |
| 59 | |
| 60 | // instanceGate is the preamble every instance verb shares, in the order the |
| 61 | // family fixed (handleAPIPromote): principal FIRST, budget second, then the |
| 62 | // deployment's own readiness. The order is load-bearing — a rate limit |
| 63 | // answered before the principal is proven would be a 429 where a 404 |
| 64 | // belongs, and "this path is rate limited" is itself the answer that an app |
| 65 | // exists here. |
| 66 | // |
| 67 | // The principal proof is apiAccountApp's, which is to say the general one: |
| 68 | // this family no longer resolves its own callers. The string it returns is |
| 69 | // the audit actor either way — an address for a person, app:<sqid>/<app> for |
| 70 | // a legacy app token exactly as before, service:<sqid>/<name> for a named |
| 71 | // service credential. |
| 72 | func (s *Server) instanceGate(w http.ResponseWriter, r *http.Request) (string, *Account, string, bool) { |
| 73 | return s.instanceGateFor(w, r, "instance") |
| 74 | } |
| 75 | |
| 76 | // instanceGateFor is instanceGate with the noun its 429 uses. The BUDGET is |
| 77 | // deliberately shared across every family that gates here — one credential |
| 78 | // doing one kind of work, and one place for the ordering to live — but a |
| 79 | // member creating sidecars should not be told they made too many "instance |
| 80 | // requests" when they hit it. Only the sentence varies; nothing about the |
| 81 | // order, the key or the limiter does. |
| 82 | func (s *Server) instanceGateFor(w http.ResponseWriter, r *http.Request, noun string) (string, *Account, string, bool) { |
| 83 | return s.instanceGateSaying(w, r, noun, mainGateCopy()) |
| 84 | } |
| 85 | |
| 86 | // instanceGateSaying is instanceGateFor with the sentences of c. |
| 87 | func (s *Server) instanceGateSaying(w http.ResponseWriter, r *http.Request, noun string, c gateCopy) (string, *Account, string, bool) { |
| 88 | principal, acct, app, ok := s.apiAccountAppSaying(w, r, s.store.HasAppIn, c) |
| 89 | if !ok { |
| 90 | return "", nil, "", false |
| 91 | } |
| 92 | // Keyed on the TOKEN, not the address (see cliInstanceBurst): the |
| 93 | // principal here is frequently an app, which has no address of its own, |
| 94 | // and a signup burst is the workload rather than the abuse. |
| 95 | if !s.cli.instances.allow(hashToken(bearer(r)), time.Now()) { |
| 96 | jsonError(w, http.StatusTooManyRequests, c.tooMany(noun)) |
| 97 | return "", nil, "", false |
| 98 | } |
| 99 | // A console with no deployment bucket configured can answer none of |
| 100 | // these. Same shape as promote's signing-key check, and in the same |
| 101 | // place: after the gate, before anything reaches for the store. |
| 102 | if s.cfg.Releases == nil { |
| 103 | jsonError(w, http.StatusServiceUnavailable, c.noBucket) |
| 104 | return "", nil, "", false |
| 105 | } |
| 106 | return principal, acct, app, true |
| 107 | } |
| 108 | |
| 109 | // instanceManifest reads the app's instances manifest and answers the |
| 110 | // caller itself when it cannot be had. ok=false means a response has |
| 111 | // already been written. |
| 112 | // |
| 113 | // CREATE is its only caller, and deliberately so: the manifest is policy |
| 114 | // about what may be BROUGHT INTO existence (which domain, which pool, which |
| 115 | // regions, which channel). Reads and deletes are about what already exists, |
| 116 | // which no version of this manifest can retract. |
| 117 | // |
| 118 | // The two failures are kept apart on purpose: ABSENCE is an answer — the |
| 119 | // feature does not exist for this app — while a read failure is the bucket |
| 120 | // being unreachable. Reporting a hiccup as "instances are not enabled" |
| 121 | // would tell an app its feature had been switched off. |
| 122 | func (s *Server) instanceManifest(ctx context.Context, w http.ResponseWriter, acct *Account, app string) (*provision.Manifest, bool) { |
| 123 | m, err := provision.GetManifest(ctx, s.cfg.Releases, acct.Sqid, app) |
| 124 | switch { |
| 125 | case err == nil: |
| 126 | return m, true |
| 127 | case errors.Is(err, os.ErrNotExist): |
| 128 | // Membership (or binding) is already proven, so this may name its |
| 129 | // cause and the operator act that fixes it. |
| 130 | jsonError(w, http.StatusConflict, |
| 131 | "instances are not enabled for "+app+" — an operator writes apps/"+ |
| 132 | acct.Sqid+"/"+app+"/instances.json (carlos instances enable)") |
| 133 | return nil, false |
| 134 | default: |
| 135 | log.Printf("console: api instances manifest %s/%s: %v", acct.Sqid, app, err) |
| 136 | jsonError(w, http.StatusBadGateway, "could not read instance settings") |
| 137 | return nil, false |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | // instanceListRecord wraps provision.Record with the console's own |
| 142 | // staged_version join (Task 8, issue #78) rather than adding the field to |
| 143 | // Record itself: Record is the wire shape CreateRecord persists to the |
| 144 | // bucket, and staged_version is never written there — it is computed fresh |
| 145 | // at every list call from status/boxes/*.json, so giving Record the field |
| 146 | // would risk a future write path serializing a stale or zero value into a |
| 147 | // stored record that this join was never meant to touch. |
| 148 | type instanceListRecord struct { |
| 149 | provision.Record |
| 150 | // StagedVersion is registry.Route.Version as last reported by whichever |
| 151 | // box's adopt pass claimed this host, read from that box's own |
| 152 | // status/boxes/<label>.json (internal/status's bucket-private Metrics |
| 153 | // block — see its package doc). Empty both for a host no box has ever |
| 154 | // reported AND for one whose last-reporting box has gone stale (see |
| 155 | // loadBoxJoin below, which asks status.BoxStale the same question |
| 156 | // the operator fleet page does): fresh-ish for as long as some box keeps |
| 157 | // reporting it, but a box that has gone silent drops OUT of the join |
| 158 | // rather than going on serving that box's last-known version as if it |
| 159 | // were current. The deploy watch does NOT read this field. |
| 160 | StagedVersion string `json:"staged_version"` |
| 161 | // Upstreams is the per-upstream probe state the box's edge currently |
| 162 | // believes for this host (status.BoxMetrics.Upstreams), joined from the |
| 163 | // same snapshots and under the same staleness rule as StagedVersion — a |
| 164 | // box that has gone silent drops out rather than going on answering for |
| 165 | // upstreams it may no longer be probing. It is what lets `carlos |
| 166 | // instances` say WHICH upstream is down without a box session, and it is |
| 167 | // omitempty because that is the honest shape for the overwhelming |
| 168 | // majority of records: only a route with more than one addr (or an |
| 169 | // explicit probe block) is probed at all, so every single-upstream record |
| 170 | // in production carries no such key and parses exactly as it always did. |
| 171 | Upstreams []status.UpstreamState `json:"upstreams,omitempty"` |
| 172 | // Health is the same one-word state the Running tab's column shows, |
| 173 | // computed by instanceHealth below from the registry row joined with the |
| 174 | // metrics sweep. Additive on the wire (a client that never heard of it |
| 175 | // keeps parsing the record it always parsed) and annotation-grade: a |
| 176 | // record whose health could not be computed still lists, reading |
| 177 | // "Setting up" rather than failing the call. |
| 178 | Health string `json:"health"` |
| 179 | // ServedBy is where this host is OBSERVED to be served (issue #337): |
| 180 | // every box whose control/edges/<box>.json names it, with that box's AWS |
| 181 | // region — the same files the edges backhaul on, so this is what the |
| 182 | // platform itself routes by, not a guess. A list, not a scalar: an |
| 183 | // everywhere-class route is genuinely served on every box. Distinct from |
| 184 | // Record.Region, which is the REQUESTED placement code ("" = |
| 185 | // unconstrained, "fleet/<name>"), and absent for a host no file names. |
| 186 | ServedBy []servedBy `json:"served_by,omitempty"` |
| 187 | // HiddenGate shadows Record.Gate for encoding/json (a shallower field of |
| 188 | // the same name wins), so the bcrypt hash never leaves the console; it |
| 189 | // is always "" and omitted. Gated is what a member may know. |
| 190 | HiddenGate string `json:"gate,omitempty"` |
| 191 | Gated bool `json:"gated"` |
| 192 | // DBBytes is the on-disk size of this instance's SQLite database (file |
| 193 | // plus -wal) as its box last measured it (status.BoxMetrics.RouteDBBytes), |
| 194 | // under the same staleness rule as StagedVersion. Absent when no fresh |
| 195 | // box has sized it: not minted yet, no database file yet, or a route |
| 196 | // whose database lives somewhere the platform does not know about. |
| 197 | DBBytes *int64 `json:"db_bytes,omitempty"` |
| 198 | // Rollout is this host's rollout as its box last reported it |
| 199 | // (zero-downtime spec §9.1), joined under StagedVersion's staleness rule. |
| 200 | // Absent when no fresh box reports one: a box from before plan 5, one |
| 201 | // with CARLOS_SWAP off, or one gone dark. The deploy watch reads it. |
| 202 | Rollout *rolloutView `json:"rollout,omitempty"` |
| 203 | // UpdateReport is this host's update report as its box last gave it |
| 204 | // (status.BoxMetrics.Updates and UpdateSettings), joined under the same |
| 205 | // staleness and tenant rules as Rollout. It is named update_report |
| 206 | // because the embedded record already owns "updates" (its own setting). |
| 207 | UpdateReport *updatesJoinView `json:"update_report,omitempty"` |
| 208 | // SettingsDefault says the record and the app's policy together ask for |
| 209 | // today's behaviour (provision.EffectiveUpdates(...).Default()), so the |
| 210 | // CLI can tell an instance on a box that ignores settings from one that |
| 211 | // never asked for any. |
| 212 | SettingsDefault bool `json:"settings_default"` |
| 213 | } |
| 214 | |
| 215 | // instanceHealth computes each record host's health word the same way the |
| 216 | // Running tab does — registry row joined with the metrics sweep — so the |
| 217 | // API, the CLI and the page can never disagree about an instance's state. |
| 218 | // A record with no registry row yet is still provisioning: "Setting up". |
| 219 | // Failures degrade to that same word rather than erroring the list: health |
| 220 | // here is an annotation on the record, not the record. |
| 221 | // |
| 222 | // The tenant boundary is the recs slice and nothing else. recs arrives |
| 223 | // already scoped to one account+app (provision.ListRecords under that app's |
| 224 | // own prefix), and every lookup below is keyed on a host drawn FROM recs — |
| 225 | // the registry read returns the whole box's routes, including other tenants', |
| 226 | // and none of them can enter hosts, unitByHost or the answer. That matters |
| 227 | // twice over: OwnerAppMetrics' allowlist is exactly the hosts and stamped |
| 228 | // units it is handed, so a foreign unit name reaching unitByHost would widen |
| 229 | // the metrics view past the caller's own app. |
| 230 | func (s *Server) instanceHealth(ctx context.Context, recs []provision.Record) map[string]string { |
| 231 | out := make(map[string]string, len(recs)) |
| 232 | for _, rec := range recs { |
| 233 | out[rec.Host] = "Setting up" |
| 234 | } |
| 235 | if s.cfg.Registry == "" { |
| 236 | // A console with no registry configured cannot know any route's |
| 237 | // state; opening "" would hand sqlite a throwaway temp DB and answer |
| 238 | // the same "Setting up" a page later, having created a file for it. |
| 239 | return out |
| 240 | } |
| 241 | reg, err := openRegistry(s.cfg.Registry) |
| 242 | if err != nil { |
| 243 | log.Printf("console: api instances health: registry %s: %v", s.cfg.Registry, err) |
| 244 | return out |
| 245 | } |
| 246 | routes, err := reg.List() |
| 247 | if err != nil { |
| 248 | log.Printf("console: api instances health: registry list: %v", err) |
| 249 | return out |
| 250 | } |
| 251 | byHost := make(map[string]registry.Route, len(routes)) |
| 252 | for _, rt := range routes { |
| 253 | byHost[rt.Host] = rt |
| 254 | } |
| 255 | // routeFor is the join reduced to the routes that ANSWER for these |
| 256 | // records, built once and used by both loops so the two can never drift. |
| 257 | routeFor := make(map[string]registry.Route, len(recs)) |
| 258 | var hosts []string |
| 259 | unitByHost := map[string]string{} |
| 260 | for _, rec := range recs { |
| 261 | rt, ok := byHost[rec.Host] |
| 262 | if !ok { |
| 263 | continue // no route minted yet: the record stays "Setting up" |
| 264 | } |
| 265 | // A row stamped to another app or account is not this record's route. |
| 266 | // The box's own reconcile pass says so — it refuses to converge such |
| 267 | // a row ("exists but was not minted by this pass — leaving it |
| 268 | // alone", reconcile.go) — and reporting it here would be worse than |
| 269 | // useless twice: the caller would read a neighbour's health word as |
| 270 | // its own instance's, and the neighbour's stamped Unit would enter |
| 271 | // unitByHost, widening OwnerAppMetrics' allowlist past this app's own |
| 272 | // units. An UNSTAMPED row (either column empty: hand-added, or minted |
| 273 | // before the column existed) still joins — refusing those would read |
| 274 | // live instances as forever "Setting up". |
| 275 | if (rt.App != "" && rt.App != rec.App) || (rt.Account != "" && rt.Account != rec.Account) { |
| 276 | continue |
| 277 | } |
| 278 | routeFor[rec.Host] = rt |
| 279 | hosts = append(hosts, rec.Host) |
| 280 | if rt.Unit != "" { |
| 281 | unitByHost[rec.Host] = rt.Unit |
| 282 | } |
| 283 | } |
| 284 | for h, word := range s.healthByHost(ctx, routeFor) { |
| 285 | out[h] = word |
| 286 | } |
| 287 | return out |
| 288 | } |
| 289 | |
| 290 | // healthByHost is the metrics half of a health answer, shared by |
| 291 | // instanceHealth (records joined to routes) and handleAPIRoutes (routes |
| 292 | // directly, issue #64) so the two — and the Running tab, which uses the |
| 293 | // same healthWord — can never disagree about an instance's state. |
| 294 | // |
| 295 | // routeFor arrives ALREADY SCOPED to the caller's own account and app. |
| 296 | // This function performs no tenant check of its own and must never be |
| 297 | // handed an unfiltered registry listing: OwnerAppMetrics' allowlist is |
| 298 | // exactly the hosts and stamped units it is given, so a foreign unit name |
| 299 | // reaching unitByHost below would widen the metrics view past the caller's |
| 300 | // own app. The scoping lives in each caller, beside the reason that |
| 301 | // caller's rows are the caller's to see. |
| 302 | func (s *Server) healthByHost(ctx context.Context, routeFor map[string]registry.Route) map[string]string { |
| 303 | out := make(map[string]string, len(routeFor)) |
| 304 | hosts := make([]string, 0, len(routeFor)) |
| 305 | unitByHost := map[string]string{} |
| 306 | for h, rt := range routeFor { |
| 307 | hosts = append(hosts, h) |
| 308 | if rt.Unit != "" { |
| 309 | unitByHost[h] = rt.Unit |
| 310 | } |
| 311 | } |
| 312 | am, err := s.OwnerAppMetrics(ctx, hosts, unitByHost) |
| 313 | if err != nil { |
| 314 | // An unreadable sweep is not knowledge that these instances are |
| 315 | // down: every host reads as not-reporting below, which healthWord |
| 316 | // turns into "No readings" or "Not responding" per expectsReadings — |
| 317 | // the same answer the page gives when the sweep is unavailable. |
| 318 | log.Printf("console: health by host: metrics: %v", err) |
| 319 | am = nil |
| 320 | } |
| 321 | rep := make(map[string]bool, len(am)) |
| 322 | for _, m := range am { |
| 323 | if m.Reporting() { |
| 324 | rep[m.Host] = true |
| 325 | } |
| 326 | } |
| 327 | for h, rt := range routeFor { |
| 328 | status := rt.Status |
| 329 | if status == "" { |
| 330 | status = "ready" // registry contract: empty status means ready |
| 331 | } |
| 332 | out[h] = healthWord(rt.Kind, status, rt.Hibernate, rep[h], |
| 333 | expectsReadings(rt.Provisioned, rt.Unit, rt.Backing)) |
| 334 | } |
| 335 | return out |
| 336 | } |
| 337 | |
| 338 | // boxJoin is everything the list annotates a record with from the FLEET's own |
| 339 | // snapshots, gathered in one pass: the staged version and the per-upstream |
| 340 | // probe table, both host-keyed. |
| 341 | // |
| 342 | // One struct and one loader rather than two functions, because the two joins |
| 343 | // must vote on staleness with the same reading of the same objects: a second |
| 344 | // LoadBoxes call could see a box go stale between them and answer a record's |
| 345 | // version from a box whose upstream table it had just refused, which is a |
| 346 | // disagreement no reader could diagnose. |
| 347 | type boxJoin struct { |
| 348 | versions map[string]string |
| 349 | serving map[string]string |
| 350 | upstreams map[string][]status.UpstreamState |
| 351 | dbBytes map[string]int64 |
| 352 | rollouts map[string]rolloutView |
| 353 | updates map[string]updatesJoinView |
| 354 | } |
| 355 | |
| 356 | // updatesJoinView is one host's update report as a member sees it. account |
| 357 | // and app are the report's tenant: the entry's own (status.HostUpdates.Account |
| 358 | // and .App) when it names one, else the one the SAME box's rollout report |
| 359 | // gives for the host (status.HostRollout.Account and .App), else none. They |
| 360 | // decide whether the report may be joined (update). |
| 361 | type updatesJoinView struct { |
| 362 | Desired string `json:"desired,omitempty"` |
| 363 | Reason string `json:"reason,omitempty"` |
| 364 | Detail string `json:"detail,omitempty"` |
| 365 | Baking string `json:"baking,omitempty"` |
| 366 | Major *int `json:"major,omitempty"` |
| 367 | Pending uint64 `json:"pending,omitempty"` |
| 368 | Completed uint64 `json:"completed,omitempty"` |
| 369 | CompletedFailed bool `json:"completed_failed,omitempty"` |
| 370 | // Enforced says the box applies update settings at all |
| 371 | // (status.BoxMetrics.UpdateSettings); a box from before them reports |
| 372 | // nothing, and an empty view must not read as "following its channel". |
| 373 | Enforced bool `json:"enforced"` |
| 374 | account, app string |
| 375 | } |
| 376 | |
| 377 | // update is the join's answer for host, or nil when the box has nothing on |
| 378 | // it or names another tenant for it. The cross-tenant rule is rollout's: a |
| 379 | // box's report about a host never surfaces on another tenant's list or page. |
| 380 | // A report is this account's and this app's only when its tenant is known |
| 381 | // and is this one (Astra r1, R18). Owning the hostname now is not enough: a |
| 382 | // snapshot stays fresh for minutes, and the host may have been another |
| 383 | // tenant's when the box wrote it, on a platform box as on a fleet box. A |
| 384 | // report with no known tenant yields whether the box enforces settings and |
| 385 | // nothing else, so an old box still reads "settings not enforced" and nothing |
| 386 | // about any instance crosses. |
| 387 | func (j boxJoin) update(host, acct, app string) *updatesJoinView { |
| 388 | v, ok := j.updates[host] |
| 389 | if !ok { |
| 390 | return nil |
| 391 | } |
| 392 | if v.account == "" && v.app == "" { |
| 393 | return &updatesJoinView{Enforced: v.Enforced} |
| 394 | } |
| 395 | if v.account != acct || v.app != app { |
| 396 | return nil |
| 397 | } |
| 398 | return &v |
| 399 | } |
| 400 | |
| 401 | // rolloutView is one host's rollout as a member sees it. account and app are |
| 402 | // the report's own, kept so the join can refuse to cross tenants (rollout). |
| 403 | type rolloutView struct { |
| 404 | Box string `json:"box"` |
| 405 | Installed string `json:"installed"` |
| 406 | Serving string `json:"serving,omitempty"` |
| 407 | Good string `json:"good,omitempty"` |
| 408 | Awake bool `json:"awake"` |
| 409 | Attempt string `json:"attempt,omitempty"` |
| 410 | AttemptBuild string `json:"attempt_build,omitempty"` |
| 411 | HeldBy string `json:"held_by,omitempty"` |
| 412 | Reason string `json:"reason,omitempty"` |
| 413 | ConfigPending bool `json:"config_pending,omitempty"` |
| 414 | // Transient is the box's word that the host's process is starting or |
| 415 | // stopping (status.HostRollout.Transient): neither asleep nor awake. |
| 416 | Transient bool `json:"transient,omitempty"` |
| 417 | // SchedulerIdle is the box's scheduler for this app: nothing queued, |
| 418 | // running or settling, first-wake observations included (spec §9.1's |
| 419 | // "a box is done" half). |
| 420 | SchedulerIdle bool `json:"scheduler_idle"` |
| 421 | // Unavailable is the reason code a box gives for being unable to arm its |
| 422 | // rollout scheduler (RolloutReport.Unavailable). When set, the view is |
| 423 | // only Box, Installed and this: the box reports no per-host state. The |
| 424 | // one exception is a host a box that declined the flag reports held by |
| 425 | // the failure book it honours (Task 10), whose view also carries Good, |
| 426 | // the attempt, HeldBy and its epoch. |
| 427 | Unavailable string `json:"unavailable,omitempty"` |
| 428 | account, app string |
| 429 | // heldEpoch is the box's epoch for HeldBy's restriction |
| 430 | // (status.HostRollout.HeldEpoch), and heldBook the failure book it |
| 431 | // counts in (HeldBook, final review round 1, finding 3). Neither is |
| 432 | // shown: the retry doors sign both into their grant (retryHostBuild). |
| 433 | heldEpoch uint64 |
| 434 | heldBook string |
| 435 | // retryV is the retry protocol the box takes (status.HostRollout.RetryV): |
| 436 | // 0 is a box built before the host-retry log, which reads no grant |
| 437 | // there, so the retry doors write it none (retryHostBuild). |
| 438 | retryV int |
| 439 | // refused marks a view built from an Unavailable box, which has no |
| 440 | // account or app to check (hosts are unique; the caller filtered by app). |
| 441 | refused bool |
| 442 | } |
| 443 | |
| 444 | // rollout is the join's answer for host, or nil unless the report is this |
| 445 | // account's and this app's: a report is the box's word about a route, and a |
| 446 | // route stamped to another app is not this record's (instanceHealth's rule). |
| 447 | func (j boxJoin) rollout(host, acct, app string) *rolloutView { |
| 448 | v, ok := j.rollouts[host] |
| 449 | if !ok || (!v.refused && (v.account != acct || v.app != app)) { |
| 450 | return nil |
| 451 | } |
| 452 | return &v |
| 453 | } |
| 454 | |
| 455 | // dbSize is the join's database size for host, nil when no fresh box |
| 456 | // reported one — so the wire field is absent rather than a 0 that reads as |
| 457 | // an empty database. |
| 458 | func (j boxJoin) dbSize(host string) *int64 { |
| 459 | n, ok := j.dbBytes[host] |
| 460 | if !ok { |
| 461 | return nil |
| 462 | } |
| 463 | return &n |
| 464 | } |
| 465 | |
| 466 | // loadBoxJoin builds boxJoin from every NON-STALE box's own |
| 467 | // private Metrics block (status/boxes/*.json), through status.LoadBoxes — |
| 468 | // the same loader the operator fleet page's HostRows reads through |
| 469 | // (internal/console/view.go) — and status.BoxStale, the fleet's one |
| 470 | // staleness rule (internal/status/fleet.go: "Every caller ... asks this |
| 471 | // function, never its own subtraction"). A box that stopped reporting is |
| 472 | // EXCLUDED from the join entirely rather than contributing its last-known |
| 473 | // reading: the alternative would let a route adopted a week ago on a now- |
| 474 | // dark box keep answering staged_version queries as if it were as current |
| 475 | // as a box that reported thirty seconds ago, which is exactly the kind of |
| 476 | // reassuring-but-wrong answer internal/status's whole design refuses to |
| 477 | // give (see BoxStale's own doc). now is passed in, not read internally, so |
| 478 | // this and BuildFleet/HostRows always vote on staleness using the request's |
| 479 | // own clock rather than three independent time.Now() calls that could |
| 480 | // disagree at a period boundary. |
| 481 | // |
| 482 | // A store that cannot be listed at all — not one corrupt object, the whole |
| 483 | // read failing — degrades to "no staged versions known" rather than |
| 484 | // answering a 500: this join is best-effort observability on top of the |
| 485 | // list, never load-bearing for it. A nil store (a console with no |
| 486 | // deployment bucket configured) is not reachable via handleAPIInstanceList |
| 487 | // today — instanceGate already refuses those with a 503 before this ever |
| 488 | // runs — but the nil check is kept here too so this helper degrades safely |
| 489 | // for any future caller that reaches it before that gate. |
| 490 | func loadBoxJoin(ctx context.Context, s release.Store, now time.Time) boxJoin { |
| 491 | j, _ := loadBoxJoinComplete(ctx, s, now) |
| 492 | return j |
| 493 | } |
| 494 | |
| 495 | // loadBoxJoinComplete is loadBoxJoin that also says whether every box's |
| 496 | // snapshot was read (status.LoadBoxesComplete): false when the listing |
| 497 | // failed, was capped, or skipped an object it could not read. The join is |
| 498 | // still best-effort; completeness is what lets a reader tell "no box reports |
| 499 | // this host" from "the report may be the one that was not read" (Astra plan |
| 500 | // 6 R1 #4: the deploy watch must not URL-poll on the second). |
| 501 | func loadBoxJoinComplete(ctx context.Context, s release.Store, now time.Time) (boxJoin, bool) { |
| 502 | if s == nil { |
| 503 | return boxJoin{}, false |
| 504 | } |
| 505 | boxes, complete, err := status.LoadBoxesComplete(ctx, s) |
| 506 | if err != nil { |
| 507 | log.Printf("console: api instances: could not load status boxes for the list join: %v", err) |
| 508 | return boxJoin{}, false |
| 509 | } |
| 510 | return joinBoxes(boxes, now), complete |
| 511 | } |
| 512 | |
| 513 | // loadRolloutJoin is the box join the instance list, the instance page and |
| 514 | // the host-retry door share (Astra plan 6 R1 #6), so the three never |
| 515 | // disagree about a host's rollout: the platform's boxes (status/boxes/), |
| 516 | // then the rollout reports of the customer-fleet boxes of THIS account's |
| 517 | // fleets (fleets/<name>/status/boxes/, the prefix systemBoxes reads). A |
| 518 | // fleet another account owns is never read: fleet names are one flat |
| 519 | // namespace across accounts, and its boxes are written by machines this |
| 520 | // account does not control. boxJoin.rollout still checks the report's own |
| 521 | // account and app. |
| 522 | // |
| 523 | // Hosts are assumed unique to one box. When two boxes name the same host (a |
| 524 | // move in flight, a report not yet stale), the last platform box read wins, |
| 525 | // as loadBoxJoin always did, and a platform box wins over any fleet box, |
| 526 | // then the first fleet box read wins: the operator's boxes are the ones this |
| 527 | // console trusts most. |
| 528 | // |
| 529 | // complete is false when a read failed on a source that could hold the |
| 530 | // reports in question: any platform box snapshot, and the fleets relevant |
| 531 | // names (Astra plan 6 R2: an unreadable snapshot on a fleet none of the |
| 532 | // app's instances is placed on says nothing about the app, and must not |
| 533 | // block its deploy watch's fallback). relevant nil counts every fleet of the |
| 534 | // account. A registry that cannot be read counts only when relevant names |
| 535 | // some fleet. A console with no route registry, or none yet, has no fleets |
| 536 | // and is complete on its platform boxes alone. Every fleet of the account is |
| 537 | // still READ, whatever relevant says: completeness is scoped, the join's |
| 538 | // content is not. |
| 539 | func (s *Server) loadRolloutJoin(ctx context.Context, acct string, now time.Time, relevant map[string]bool) (boxJoin, bool) { |
| 540 | j, complete := loadBoxJoinComplete(ctx, s.cfg.Releases, now) |
| 541 | if s.cfg.Releases == nil || s.cfg.Registry == "" { |
| 542 | return j, complete |
| 543 | } |
| 544 | counts := func(fleet string) bool { return relevant == nil || relevant[fleet] } |
| 545 | unreadable := relevant == nil || len(relevant) > 0 |
| 546 | reg, err := s.openPassRegistry() |
| 547 | switch { |
| 548 | case errors.Is(err, os.ErrNotExist): |
| 549 | return j, complete // no registry file yet: no fleets |
| 550 | case err != nil: |
| 551 | log.Printf("console: rollout join %s: open registry: %v", acct, err) |
| 552 | return j, complete && !unreadable |
| 553 | } |
| 554 | fleets, err := reg.ListFleets() |
| 555 | if err != nil { |
| 556 | log.Printf("console: rollout join %s: list fleets: %v", acct, err) |
| 557 | return j, complete && !unreadable |
| 558 | } |
| 559 | for _, f := range fleets { |
| 560 | if f.Account != acct { |
| 561 | continue |
| 562 | } |
| 563 | boxes, fleetComplete, err := status.LoadBoxesComplete(ctx, release.NewPrefixStore(s.cfg.Releases, "fleets/"+f.Name+"/")) |
| 564 | if err != nil { |
| 565 | log.Printf("console: rollout join %s: fleet %s boxes: %v", acct, f.Name, err) |
| 566 | complete = complete && !counts(f.Name) |
| 567 | continue |
| 568 | } |
| 569 | complete = complete && (fleetComplete || !counts(f.Name)) |
| 570 | j.joinFleetRollouts(boxes, now) |
| 571 | } |
| 572 | return j, complete |
| 573 | } |
| 574 | |
| 575 | // recordFleets is the set of customer fleets that could hold reports for |
| 576 | // recs: the fleet each record is placed on (its Region, "fleet/<name>"), |
| 577 | // and either end of a move in flight. Never nil, so an app with no |
| 578 | // fleet-placed record counts no fleet at all. |
| 579 | func recordFleets(recs []provision.Record) map[string]bool { |
| 580 | out := map[string]bool{} |
| 581 | for _, rec := range recs { |
| 582 | places := []string{rec.Region} |
| 583 | if rec.Move != nil { |
| 584 | places = append(places, rec.Move.From, rec.Move.To) |
| 585 | } |
| 586 | for _, p := range places { |
| 587 | if f, ok := registry.FleetPlacement(p); ok { |
| 588 | out[f] = true |
| 589 | } |
| 590 | } |
| 591 | } |
| 592 | return out |
| 593 | } |
| 594 | |
| 595 | // joinBoxes is loadBoxJoin's fold over snapshots already read, so a page that |
| 596 | // has its boxes reads them once. |
| 597 | func joinBoxes(boxes []status.Box, now time.Time) boxJoin { |
| 598 | j := boxJoin{versions: map[string]string{}, serving: map[string]string{}, upstreams: map[string][]status.UpstreamState{}, |
| 599 | dbBytes: map[string]int64{}, rollouts: map[string]rolloutView{}} |
| 600 | for _, b := range boxes { |
| 601 | if b.Metrics == nil || status.BoxStale(b, now, status.SnapshotPeriod) { |
| 602 | continue |
| 603 | } |
| 604 | for host, v := range b.Metrics.RouteVersions { |
| 605 | j.versions[host] = v |
| 606 | } |
| 607 | for host, v := range b.Metrics.RouteServing { |
| 608 | j.serving[host] = v |
| 609 | } |
| 610 | for host, ups := range b.Metrics.Upstreams { |
| 611 | j.upstreams[host] = ups |
| 612 | } |
| 613 | for host, n := range b.Metrics.RouteDBBytes { |
| 614 | j.dbBytes[host] = n |
| 615 | } |
| 616 | j.foldRollout(b, true) |
| 617 | j.foldUpdates(b, true) |
| 618 | } |
| 619 | return j |
| 620 | } |
| 621 | |
| 622 | // foldUpdates adds box b's update report to the join, under foldRollout's |
| 623 | // override rule (a fleet box never replaces a host a platform box already |
| 624 | // reported). Every host the box serves gets a view saying whether the box |
| 625 | // enforces settings; the hosts it reports on carry the report. The tenant is |
| 626 | // the report's own, else whatever the same box's rollout report names for |
| 627 | // the host (updatesJoinView). |
| 628 | func (j *boxJoin) foldUpdates(b status.Box, override bool) { |
| 629 | m := b.Metrics |
| 630 | enforced := m.UpdateSettings >= status.UpdateSettingsProtocol |
| 631 | if j.updates == nil { |
| 632 | j.updates = map[string]updatesJoinView{} |
| 633 | } |
| 634 | put := func(host string) { |
| 635 | if _, had := j.updates[host]; had && !override { |
| 636 | return |
| 637 | } |
| 638 | v := updatesJoinView{Enforced: enforced} |
| 639 | if m.Rollout != nil { |
| 640 | if hr, ok := m.Rollout.Hosts[host]; ok { |
| 641 | v.account, v.app = hr.Account, hr.App |
| 642 | } |
| 643 | } |
| 644 | if u, ok := m.Updates[host]; ok { |
| 645 | v.Desired, v.Reason, v.Detail, v.Baking, v.Major = u.Desired, u.Reason, u.Detail, u.Baking, u.Major |
| 646 | v.Pending, v.Completed, v.CompletedFailed = u.Pending, u.Completed, u.CompletedFailed |
| 647 | if u.Account != "" || u.App != "" { |
| 648 | v.account, v.app = u.Account, u.App |
| 649 | } |
| 650 | } |
| 651 | j.updates[host] = v |
| 652 | } |
| 653 | for host := range m.RouteVersions { |
| 654 | put(host) |
| 655 | } |
| 656 | for host := range m.Updates { |
| 657 | put(host) |
| 658 | } |
| 659 | } |
| 660 | |
| 661 | // foldRollout adds box b's rollout report to the join. override false keeps |
| 662 | // a host the join already has: the caller folds platform boxes first, and a |
| 663 | // fleet box's word about a host never replaces a platform box's. |
| 664 | func (j *boxJoin) foldRollout(b status.Box, override bool) { |
| 665 | ro := b.Metrics.Rollout |
| 666 | if ro == nil { |
| 667 | return |
| 668 | } |
| 669 | if j.rollouts == nil { |
| 670 | j.rollouts = map[string]rolloutView{} |
| 671 | } |
| 672 | put := func(host string, v rolloutView) { |
| 673 | if _, had := j.rollouts[host]; had && !override { |
| 674 | return |
| 675 | } |
| 676 | j.rollouts[host] = v |
| 677 | } |
| 678 | if ro.Unavailable != "" { |
| 679 | // The scheduler could not arm. Say so for every host this box |
| 680 | // serves, from the version it reports for it. A box that declined |
| 681 | // the flag but honours its failure book also reports the hosts the |
| 682 | // book holds (Task 10, plan review #5): those keep the code, and |
| 683 | // carry the hold and its epoch so a retry can be signed for them |
| 684 | // (retryHostBuild), under the report's own account and app. |
| 685 | for host, v := range b.Metrics.RouteVersions { |
| 686 | if _, held := ro.Hosts[host]; held { |
| 687 | continue |
| 688 | } |
| 689 | put(host, rolloutView{Box: b.Label, Installed: v, Unavailable: ro.Unavailable, refused: true}) |
| 690 | } |
| 691 | for host, hr := range ro.Hosts { |
| 692 | put(host, rolloutView{Box: b.Label, Installed: hr.Installed, Good: hr.Good, Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild, |
| 693 | HeldBy: hr.HeldBy, Reason: hr.Reason, Unavailable: ro.Unavailable, account: hr.Account, app: hr.App, heldEpoch: hr.HeldEpoch, heldBook: hr.HeldBook, retryV: hr.RetryV}) |
| 694 | } |
| 695 | return |
| 696 | } |
| 697 | // Plan 5 marks every host a failure record holds as failed. The |
| 698 | // record's own Host is the one the build failed on; the rest are held |
| 699 | // by it (spec §9.1), so their attempt is cleared and HeldBy kept. A |
| 700 | // hold-mode host left down (AttemptDown) stays as it is. |
| 701 | failedOn := map[[4]string]bool{} |
| 702 | for _, f := range ro.Failures { |
| 703 | failedOn[[4]string{f.Account, f.App, f.Build, f.Host}] = true |
| 704 | } |
| 705 | for host, hr := range ro.Hosts { |
| 706 | if hr.Attempt == status.AttemptFailed && !failedOn[[4]string{hr.Account, hr.App, hr.AttemptBuild, host}] { |
| 707 | hr.Attempt, hr.AttemptBuild = "", "" |
| 708 | } |
| 709 | a := ro.Apps[hr.Account+"/"+hr.App] |
| 710 | put(host, rolloutView{ |
| 711 | Box: b.Label, Installed: hr.Installed, Serving: hr.Serving, Good: hr.Good, Awake: hr.Awake, |
| 712 | Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild, HeldBy: hr.HeldBy, Reason: hr.Reason, |
| 713 | ConfigPending: hr.ConfigPending, Transient: hr.Transient, SchedulerIdle: a.Queued == 0 && a.Running == 0 && a.Settling == 0, |
| 714 | account: hr.Account, app: hr.App, heldEpoch: hr.HeldEpoch, heldBook: hr.HeldBook, retryV: hr.RetryV, |
| 715 | }) |
| 716 | } |
| 717 | } |
| 718 | |
| 719 | // joinFleetRollouts folds customer-fleet boxes into the join: their rollout |
| 720 | // reports only, and never over a host a platform box already reported. Their |
| 721 | // route versions, upstreams and database sizes stay out, so the list's other |
| 722 | // columns are the platform join's, as before. Staleness is the platform's |
| 723 | // rule (status.BoxStale). |
| 724 | func (j *boxJoin) joinFleetRollouts(boxes []status.Box, now time.Time) { |
| 725 | for _, b := range boxes { |
| 726 | if b.Metrics == nil || status.BoxStale(b, now, status.SnapshotPeriod) { |
| 727 | continue |
| 728 | } |
| 729 | j.foldRollout(b, false) |
| 730 | j.foldUpdates(b, false) |
| 731 | } |
| 732 | } |
| 733 | |
| 734 | // handleAPIInstanceList answers `carlos instances list`: this app's records, |
| 735 | // capped and honest about the cap, each carrying its best-effort |
| 736 | // staged_version (Task 8, issue #78) — fresh-ish while some box keeps |
| 737 | // reporting the host, dropped rather than stale once none does — and its |
| 738 | // health word, plus the same q/health/limit/cursor controls the Running tab |
| 739 | // has, applied SERVER-side so the CLI and the page filter identically. |
| 740 | // |
| 741 | // Every addition is additive on the wire and defaults to today's answer: no |
| 742 | // params means the full `instanceListMax` page, `instances` and `truncated` |
| 743 | // exactly as before, with `health` on each record and `total`/`counts`/ |
| 744 | // `next_cursor` alongside. Counts cover every record regardless of the |
| 745 | // filter — they are the whole app's health story, and a filtered count could |
| 746 | // not lead anywhere. |
| 747 | func (s *Server) handleAPIInstanceList(w http.ResponseWriter, r *http.Request) { |
| 748 | _, acct, app, ok := s.instanceGate(w, r) |
| 749 | if !ok { |
| 750 | return |
| 751 | } |
| 752 | ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| 753 | defer cancel() |
| 754 | recs, truncated, skipped, err := provision.ListRecordsComplete(ctx, s.cfg.Releases, acct.Sqid, app, instanceListMax) |
| 755 | if err != nil { |
| 756 | log.Printf("console: api instances list %s/%s: %v", acct.Sqid, app, err) |
| 757 | jsonError(w, http.StatusBadGateway, "could not list instances") |
| 758 | return |
| 759 | } |
| 760 | relevant := recordFleets(recs) |
| 761 | if len(skipped) > 0 { |
| 762 | // A skipped record's placement is unknown, so every fleet the account |
| 763 | // has, and the platform, counts toward completeness (nil means all). |
| 764 | // Astra plan 6 R3: an unreadable record must not drop its fleet. |
| 765 | log.Printf("console: api instances list %s/%s: %d record(s) unreadable %v; every fleet counts toward the rollout join", |
| 766 | acct.Sqid, app, len(skipped), skipped) |
| 767 | relevant = nil |
| 768 | } |
| 769 | join, joinComplete := s.loadRolloutJoin(ctx, acct.Sqid, time.Now(), relevant) |
| 770 | served := loadEdgesJoin(ctx, s.cfg.Releases) |
| 771 | // The app's update policy, read once: an absent manifest is no policy, |
| 772 | // and a read error is logged and treated the same. The list never fails |
| 773 | // for it; SettingsDefault is then computed from the record alone. |
| 774 | var policy *provision.UpdatePolicy |
| 775 | if m, err := provision.GetManifest(ctx, s.cfg.Releases, acct.Sqid, app); err == nil { |
| 776 | policy = m.Updates |
| 777 | } else if !errors.Is(err, os.ErrNotExist) { |
| 778 | log.Printf("console: api instances list %s/%s: update policy: %v; settings_default computed from the records alone", acct.Sqid, app, err) |
| 779 | } |
| 780 | params := r.URL.Query() |
| 781 | q := parseInstanceQuery(params) |
| 782 | // The default stays the whole cap, so a client that sends no limit gets |
| 783 | // the answer it always got. A limit outside (0, instanceListMax) is |
| 784 | // ignored rather than refused — a bad page size is not worth an error |
| 785 | // where the honest full page is right there. |
| 786 | limit := instanceListMax |
| 787 | if raw := params.Get("limit"); raw != "" { |
| 788 | if n, err := strconv.Atoi(raw); err == nil && n > 0 && n < instanceListMax { |
| 789 | limit = n |
| 790 | } |
| 791 | } |
| 792 | health := s.instanceHealth(ctx, recs) |
| 793 | counts := map[string]int{} |
| 794 | matched := make([]instanceListRecord, 0, len(recs)) |
| 795 | for _, rec := range recs { |
| 796 | h := health[rec.Host] |
| 797 | counts[h]++ |
| 798 | if q.Q != "" && !strings.Contains(rec.Host, q.Q) { |
| 799 | continue |
| 800 | } |
| 801 | if q.Channel != "" && rec.Channel != q.Channel { |
| 802 | continue |
| 803 | } |
| 804 | if q.Health != "" && h != q.Health { |
| 805 | continue |
| 806 | } |
| 807 | matched = append(matched, instanceListRecord{Record: rec, |
| 808 | StagedVersion: join.versions[rec.Host], Health: h, Upstreams: join.upstreams[rec.Host], |
| 809 | ServedBy: served[rec.Host], Gated: rec.Gate != "", DBBytes: join.dbSize(rec.Host), |
| 810 | Rollout: join.rollout(rec.Host, acct.Sqid, app), |
| 811 | UpdateReport: join.update(rec.Host, acct.Sqid, app), |
| 812 | SettingsDefault: provision.EffectiveUpdates(rec, policy).Default()}) |
| 813 | } |
| 814 | total := len(matched) |
| 815 | // provision.ListRecords returns records host-sorted and nothing above |
| 816 | // re-orders them, which is the whole basis of the cursor: "resume after |
| 817 | // this host" only means one page in a stable order. |
| 818 | start := 0 |
| 819 | if after, ok := decodeInstanceCursor(q.Cursor); ok { |
| 820 | for i, rec := range matched { |
| 821 | if rec.Host > after { |
| 822 | start = i |
| 823 | break |
| 824 | } |
| 825 | start = i + 1 |
| 826 | } |
| 827 | } |
| 828 | next := "" |
| 829 | end := start + limit |
| 830 | if end < total { |
| 831 | next = encodeInstanceCursor(matched[end-1].Host) |
| 832 | } else { |
| 833 | end = total |
| 834 | } |
| 835 | answer := map[string]any{ |
| 836 | "instances": matched[start:end], "truncated": truncated, |
| 837 | "total": total, "counts": counts, "next_cursor": next, |
| 838 | } |
| 839 | // Additive and only when true, so a complete join's answer is the bytes |
| 840 | // it always was: the box join behind the rollout views could not read |
| 841 | // every box's report, so a record with no rollout may yet have one. |
| 842 | if !joinComplete { |
| 843 | answer["rollout_incomplete"] = true |
| 844 | } |
| 845 | if s.cfg.HostClaim { |
| 846 | pending, err := s.claimPendingViews(ctx, acct.Sqid, app, "instance") |
| 847 | if err != nil { |
| 848 | writeClaimResponseError(w, err) |
| 849 | return |
| 850 | } |
| 851 | answer["operations"] = pending |
| 852 | } |
| 853 | writeJSON(w, http.StatusOK, answer) |
| 854 | } |
| 855 | |
| 856 | // handleAPIInstanceGet answers one record. A host with no record is the |
| 857 | // byte-identical 404 — a record's existence is not a thing to probe for |
| 858 | // either, and this way "no such instance" and "no such app" read alike. |
| 859 | func (s *Server) handleAPIInstanceGet(w http.ResponseWriter, r *http.Request) { |
| 860 | _, acct, app, ok := s.instanceGate(w, r) |
| 861 | if !ok { |
| 862 | return |
| 863 | } |
| 864 | host := r.PathValue("host") |
| 865 | // Shape-checked before the key builder, not after. A {host} segment |
| 866 | // arrives percent-decoded, so "..%2f..%2fchannels" reaches this handler |
| 867 | // as a path with separators in it; RecordKey would happily join it. A |
| 868 | // host that cannot name a record is simply not found — the same answer |
| 869 | // a host that names no record gets. |
| 870 | if !provision.ValidHostShape(host) { |
| 871 | http.NotFound(w, r) |
| 872 | return |
| 873 | } |
| 874 | ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| 875 | defer cancel() |
| 876 | rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host) |
| 877 | if errors.Is(err, os.ErrNotExist) { |
| 878 | http.NotFound(w, r) |
| 879 | return |
| 880 | } |
| 881 | if err != nil { |
| 882 | log.Printf("console: api instances get %s/%s/%s: %v", acct.Sqid, app, host, err) |
| 883 | jsonError(w, http.StatusBadGateway, "could not read the instance") |
| 884 | return |
| 885 | } |
| 886 | writeJSON(w, http.StatusOK, instanceGetRecord{Record: *rec, |
| 887 | ServedBy: loadEdgesJoin(ctx, s.cfg.Releases)[host], Gated: rec.Gate != ""}) |
| 888 | } |
| 889 | |
| 890 | // instanceGetRecord is the single-record answer: the stored record plus the |
| 891 | // same observed-placement join the list carries (issue #337). Only that — |
| 892 | // the get has never carried staged_version/health, and growing it to the |
| 893 | // list's full row is a separate decision from telling a consumer where one |
| 894 | // instance lives. |
| 895 | type instanceGetRecord struct { |
| 896 | provision.Record |
| 897 | ServedBy []servedBy `json:"served_by,omitempty"` |
| 898 | // HiddenGate shadows Record.Gate for encoding/json (a shallower field of |
| 899 | // the same name wins), so the bcrypt hash never leaves the console; it |
| 900 | // is always "" and omitted. Gated is what a member may know. |
| 901 | HiddenGate string `json:"gate,omitempty"` |
| 902 | Gated bool `json:"gated"` |
| 903 | } |
| 904 | |
| 905 | // apiRecordView is a record as the write doors answer it: the whole record |
| 906 | // with the gate hash shadowed and a gated flag in its place. Every handler |
| 907 | // that used to writeJSON a bare record answers through this, so there is one |
| 908 | // place — not five — where "the hash never leaves" has to hold. |
| 909 | type apiRecordView struct { |
| 910 | provision.Record |
| 911 | HiddenGate string `json:"gate,omitempty"` |
| 912 | Gated bool `json:"gated"` |
| 913 | // PrivateStoreBucket is the bucket the create door provisioned for this |
| 914 | // instance under the app's private-storage policy (issue #412), empty on |
| 915 | // every other record and every other door. Not part of the record: it is |
| 916 | // a fact about what this one request did, and the record's own |
| 917 | // Environment is what names the store afterwards. |
| 918 | PrivateStoreBucket string `json:"private_store_bucket,omitempty"` |
| 919 | // RegionFrom and RegionName say how this create chose its region and |
| 920 | // what the place is called (spec §7): facts about this request, not |
| 921 | // part of the record. |
| 922 | RegionFrom string `json:"region_from,omitempty"` |
| 923 | RegionName string `json:"region_name,omitempty"` |
| 924 | } |
| 925 | |
| 926 | func recordView(rec provision.Record) apiRecordView { |
| 927 | return apiRecordView{Record: rec, Gated: rec.Gate != ""} |
| 928 | } |
| 929 | |
| 930 | // replicaFields is the #162 slice of the instance-create body. Role and |
| 931 | // placement travel together or not at all — "a replica without a placement" |
| 932 | // and "a placement without a replica role" are both half a decision, and |
| 933 | // each half alone produces a record some other box will act on while the |
| 934 | // writer thinks they asked for nothing. Replicas (the home's own placement |
| 935 | // set) rides the same body so one call can provision the replica AND publish |
| 936 | // where the account's replicas live. |
| 937 | type replicaFields struct { |
| 938 | Role string `json:"role,omitempty"` |
| 939 | Replicas []string `json:"replicas,omitempty"` |
| 940 | } |
| 941 | |
| 942 | // checkReplicaCreate is the create door's replica rails (#162). Returns nil |
| 943 | // when the request carries no replica fields at all — the overwhelmingly |
| 944 | // common shape, byte-identical to every create before this door existed — |
| 945 | // and a refusal when it carries half of one. The home's own placement set is |
| 946 | // canonicalized (sorted, deduped, ValidRegion-checked) in place, so the wire |
| 947 | // column never carries a non-canonical order and the reconciler's converge |
| 948 | // has nothing to re-canonicalize on the next pass. |
| 949 | func checkReplicaCreate(req replicaFields, rec *provision.Record) *doorRefusal { |
| 950 | if req.Role == "" && len(req.Replicas) == 0 { |
| 951 | return nil |
| 952 | } |
| 953 | // A replica's placement IS its region — one placement, the create's own |
| 954 | // --region — so an empty one is half a replica: a mirror with nowhere to |
| 955 | // live. (The armed-placement gate below refuses one that names no real |
| 956 | // box; this refuses one that names nothing at all.) |
| 957 | if req.Role != "" && rec.Region == "" { |
| 958 | return &doorRefusal{http.StatusBadRequest, |
| 959 | "role replica needs a placement — pass the placement code the replica lives in as the create's region"} |
| 960 | } |
| 961 | // A replica does not carry a replica set of its own: the set is the |
| 962 | // HOME's opt-in naming where its replicas live, and a replica of a |
| 963 | // replica is not a shape the platform carries. |
| 964 | if req.Role != "" && len(req.Replicas) > 0 { |
| 965 | return &doorRefusal{http.StatusBadRequest, |
| 966 | "role replica with replicas: the placement set is the HOME record's opt-in — a replica does not carry one"} |
| 967 | } |
| 968 | // The home's own placement set is canonicalized (sorted, deduped, |
| 969 | // ValidRegion-checked) in place, so the wire column never carries a |
| 970 | // non-canonical order and the reconciler's converge has nothing to |
| 971 | // re-canonicalize on the next pass. |
| 972 | if len(req.Replicas) > 0 { |
| 973 | norm, err := provision.ValidReplicas(req.Replicas) |
| 974 | if err != nil { |
| 975 | return &doorRefusal{http.StatusBadRequest, err.Error()} |
| 976 | } |
| 977 | rec.Replicas = norm |
| 978 | } |
| 979 | return nil |
| 980 | } |
| 981 | |
| 982 | // handleAPIInstanceCreate is the point of the feature: one create-only |
| 983 | // write to the bucket, in spec §7's refusal order — gate, budget, manifest, |
| 984 | // host, region, channel, write. Every field the record carries that policy |
| 985 | // governs (pool, and the channel default) is stamped from the MANIFEST, not |
| 986 | // from the request: the body below has no pool field to lie in at all. |
| 987 | func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) { |
| 988 | principal, acct, app, ok := s.instanceGate(w, r) |
| 989 | if !ok { |
| 990 | return |
| 991 | } |
| 992 | // A PERSON must be able to write the control plane's own instance |
| 993 | // record — that is the act that makes `carlos deploy -app console` |
| 994 | // resolve a host at all, and refusing it here would close the door this |
| 995 | // family exists to open. A SERVICE credential must not: the record it |
| 996 | // would write is the thing the reconciler then owns, and ownership of |
| 997 | // the console's route is not a CI token's to acquire (controlplane.go). |
| 998 | // |
| 999 | // Straight after instanceGate, which is to say after the family's |
| 1000 | // byte-identical 404 and before the manifest read: a non-member must |
| 1001 | // never learn from this door that an app here carries the flag. |
| 1002 | // principalFrom's ok is discarded for the reason handleAPIPromote's |
| 1003 | // comment gives — cliRoute stashed the principal before the handler |
| 1004 | // started, so it is already present. |
| 1005 | p, _ := principalFrom(r.Context()) |
| 1006 | if s.refuseServiceControlPlane(w, p, acct.ID, app, "create its instance records") { |
| 1007 | return |
| 1008 | } |
| 1009 | ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| 1010 | defer cancel() |
| 1011 | m, ok := s.instanceManifest(ctx, w, acct, app) |
| 1012 | if !ok { |
| 1013 | return |
| 1014 | } |
| 1015 | var req struct { |
| 1016 | Host string `json:"host"` |
| 1017 | Region string `json:"region"` |
| 1018 | Channel string `json:"channel"` |
| 1019 | Message string `json:"message"` |
| 1020 | // The route SHAPE fields (route-records design §3, HA design §II). |
| 1021 | // All optional, all defaulting to the exact route this door minted |
| 1022 | // before they existed, and all gated by provision.ValidateShape |
| 1023 | // below — the same door the reconcile pass re-checks them at, because |
| 1024 | // neither side trusts the other about what is in the bucket. |
| 1025 | // |
| 1026 | // What is deliberately NOT here: pool (manifest policy), hibernate |
| 1027 | // and unit (operator columns with their own verbs), and anything |
| 1028 | // that could widen which HOST this app may claim — the suffix check |
| 1029 | // below is unchanged, so a "kind":"static" body cannot reach for a |
| 1030 | // hostname outside the app's own domain. |
| 1031 | // |
| 1032 | // Environment WAS on that list until 2026-09-04, and the omission |
| 1033 | // had a cost worth recording. Config in CARLOS is per-app, so a |
| 1034 | // second instance standing alongside a live one inherits the |
| 1035 | // default bundle — including whatever single origin the app's |
| 1036 | // framework compares the request's Origin header against. That |
| 1037 | // makes every POST to the new host fail closed: sign-in, magic |
| 1038 | // links, WebAuthn RP ID, the cookie Secure flag, all keyed off a |
| 1039 | // scalar that names the OTHER host. `carlos canary` mints exactly |
| 1040 | // such a route, so without this field the verb hands you a live |
| 1041 | // hostname that cannot accept a form submission for any app that |
| 1042 | // carries config, which is most of them. |
| 1043 | // |
| 1044 | // It is not an operator column in the way pool and unit are. Those |
| 1045 | // govern host resources the member has no standing over; a config |
| 1046 | // bundle is the app's own, written by the app's own `carlos env |
| 1047 | // --environment`, and binding one of an app's routes to one of its |
| 1048 | // own bundles is squarely inside what a member already controls. |
| 1049 | // ValidateShape (provision.go) checks the name, and the same |
| 1050 | // control-plane guard the PATCH door applies is applied above. |
| 1051 | Kind string `json:"kind"` |
| 1052 | Environment string `json:"environment"` |
| 1053 | Addrs []string `json:"addrs"` |
| 1054 | Health *provision.Health `json:"probe"` |
| 1055 | // The replica fields (#162), embedded so their wire keys sit beside |
| 1056 | // kind/environment's — one flat body, no nesting for the caller to |
| 1057 | // discover. |
| 1058 | replicaFields |
| 1059 | // The seed doors (seeded-instances spec §4, §5.1): a create that |
| 1060 | // starts from a sibling's replica or an uploaded snapshot. Both run |
| 1061 | // every check below, then leave this handler (apiinstanceseed.go, |
| 1062 | // apiinstanceimport.go). |
| 1063 | From *seedFromReq `json:"from,omitempty"` |
| 1064 | Import *seedImportReq `json:"import,omitempty"` |
| 1065 | } |
| 1066 | if !readJSON(w, r, &req) { |
| 1067 | return |
| 1068 | } |
| 1069 | message, ok := checkedActivityMessage(w, req.Message) |
| 1070 | if !ok { |
| 1071 | return |
| 1072 | } |
| 1073 | seeded := req.From != nil || req.Import != nil |
| 1074 | if seeded { |
| 1075 | if ref := seedOneDoorRefusal(req.From != nil, req.Import != nil); ref != nil { |
| 1076 | jsonError(w, ref.Status, ref.Msg) |
| 1077 | return |
| 1078 | } |
| 1079 | if ref := seedCommonRefusal(strings.TrimSpace(req.Kind), req.Addrs, strings.TrimSpace(req.Role), req.Replicas); ref != nil { |
| 1080 | jsonError(w, ref.Status, ref.Msg) |
| 1081 | return |
| 1082 | } |
| 1083 | } |
| 1084 | host := strings.TrimSpace(req.Host) |
| 1085 | // Path-traversal defence stated as such (spec §11): this host becomes a |
| 1086 | // bucket key segment here and, on the box, the <host>.sock and |
| 1087 | // <host>.db path segments. Nothing below runs until it is one label |
| 1088 | // plus the app's own domain. |
| 1089 | if !provision.ValidHost(host, m.Domain) { |
| 1090 | jsonError(w, http.StatusBadRequest, |
| 1091 | "invalid host — an instance host is one DNS label followed by ."+m.Domain) |
| 1092 | return |
| 1093 | } |
| 1094 | // A temporary app serves one static site at its own platform address |
| 1095 | // (spec §4, ruling 5): anything else is estate an automatic purge would |
| 1096 | // have to tear down, or a host outside what its expiry drops. |
| 1097 | if apex := s.apexHost(acct.Sqid, app); strings.TrimSpace(req.Kind) != "static" || apex == "" || host != apex { |
| 1098 | if s.refuseTemporary(w, acct, app, "serve anything but a static site at "+app+"'s own platform address") { |
| 1099 | return |
| 1100 | } |
| 1101 | } |
| 1102 | // The account's platform subdomain is a SHARED namespace: every app's |
| 1103 | // apex lives at <app>.<sqid>.<AppsDomain>, and a manifest pinned there |
| 1104 | // (the member-facing enable door) makes every sibling's apex a valid |
| 1105 | // host shape for THIS app. A record at a sibling's apex would mint a |
| 1106 | // real route there first, and the domains pass would then skip the |
| 1107 | // sibling's own apex mint forever (byHost) — one app quietly serving |
| 1108 | // another's public URL (review 2026-08-06 #1). Within-account only, but |
| 1109 | // reachable by an app token, so it is refused rather than trusted to |
| 1110 | // membership: a label naming another claimed app in the account belongs |
| 1111 | // to that app. Custom-domain manifests never enter apex space (ValidHost |
| 1112 | // pinned the suffix above), so this check cannot touch them. |
| 1113 | // |
| 1114 | // "Claimed" includes an app in the TRASH. A trashed app's name is |
| 1115 | // reserved until purge — CreateAppIn and RenameAppIn both refuse it — and |
| 1116 | // so is its apex: were the check live apps only, a sibling could take the |
| 1117 | // apex while the app sat in the trash and keep it across a restore, with |
| 1118 | // the domains pass deferring to the sibling's route for ever. That is |
| 1119 | // also where `carlos email events push` sends the restored app's |
| 1120 | // recipients' addresses (discussion 153). Only a purge frees the apex. |
| 1121 | // The name arriving after the instance — a create, rename or move — is |
| 1122 | // the other way in, and apex.go closes it. |
| 1123 | // |
| 1124 | // The same question answers a second one further down (the steering |
| 1125 | // default), so it is asked once, here, and named: is this manifest |
| 1126 | // pinned to the account's platform subdomain, or to a domain of the |
| 1127 | // app's own? |
| 1128 | platformNamespace := s.cfg.AppsDomain != "" && m.Domain == acct.Sqid+"."+s.cfg.AppsDomain |
| 1129 | // siblingLabel is set when this host sits at a label that is not the |
| 1130 | // app's own — the case re-checked after the record is written, below. |
| 1131 | siblingLabel := "" |
| 1132 | if platformNamespace { |
| 1133 | if label := strings.TrimSuffix(host, "."+m.Domain); label != app { |
| 1134 | siblingLabel = label |
| 1135 | live, trashed, err := s.store.AppNameState(acct.ID, label) |
| 1136 | if err != nil { |
| 1137 | log.Printf("console: api instances create sibling check %s/%s: %v", acct.Sqid, label, err) |
| 1138 | jsonError(w, http.StatusInternalServerError, "could not check the host — try again") |
| 1139 | return |
| 1140 | } |
| 1141 | switch { |
| 1142 | case live: |
| 1143 | jsonError(w, http.StatusConflict, |
| 1144 | host+" is app "+label+"'s platform apex — create it as an instance of "+label+" instead") |
| 1145 | return |
| 1146 | case trashed: |
| 1147 | jsonError(w, http.StatusConflict, |
| 1148 | host+" is app "+label+"'s platform apex, and "+label+" is in the trash — its apex stays reserved until it is purged") |
| 1149 | return |
| 1150 | } |
| 1151 | } |
| 1152 | } |
| 1153 | // The sibling check above names apps; this one names HOSTS. Records are |
| 1154 | // keyed per app, so another app of this account may already hold the very |
| 1155 | // host this one is asking for without being called by its label: canary |
| 1156 | // "a" of app "b-c" and canary "a-b" of app "c" are both a-b-c (the flat |
| 1157 | // canary host, discussion 207). The provisioning pass would keep one |
| 1158 | // record and skip the other, so the loser would look created and serve |
| 1159 | // nothing. Refused here, for every host, on the same fail-closed terms as |
| 1160 | // the apex checks; a host this app already holds falls through to the |
| 1161 | // create's own "already exists". |
| 1162 | // |
| 1163 | // KNOWN LIMIT, filed as amadan discussion 232: this is a check, then a |
| 1164 | // write, and CreateRecord's conditional write protects only this app's |
| 1165 | // own account/app/host key. Two creates for one host from two different |
| 1166 | // apps, racing, can both pass and both be written; the provisioning pass |
| 1167 | // then keeps one and skips the other ("duplicate host"). Closing that |
| 1168 | // needs host-scoped atomic ownership, which is a gap in the instance door |
| 1169 | // that predates the flat canary host and is not built here. The |
| 1170 | // sequential case, the common one, is closed. |
| 1171 | switch hold, err := s.hostHolder(ctx, acct, host, app); { |
| 1172 | case err != nil: |
| 1173 | log.Printf("console: api instances create %s/%s: host holder check %s: %v", acct.Sqid, app, host, err) |
| 1174 | jsonError(w, http.StatusInternalServerError, "could not check the host — try again") |
| 1175 | return |
| 1176 | case hold != nil: |
| 1177 | jsonError(w, http.StatusConflict, hostHeldSentence(host, hold)) |
| 1178 | return |
| 1179 | } |
| 1180 | // An attached alias has no record and may have no route yet, so the |
| 1181 | // holder check cannot see it; its authoritative claim is one read. |
| 1182 | switch hold, err := s.hostClaimHolder(ctx, acct, host, app); { |
| 1183 | case err != nil: |
| 1184 | log.Printf("console: api instances create %s/%s: host claim check %s: %v", acct.Sqid, app, host, err) |
| 1185 | jsonError(w, http.StatusInternalServerError, "could not check the host — try again") |
| 1186 | return |
| 1187 | case hold != nil: |
| 1188 | jsonError(w, http.StatusConflict, claimHeldSentence(host, hold)) |
| 1189 | return |
| 1190 | } |
| 1191 | // A fleet-placed app (customer-fleets plan, `carlos apps place`) governs |
| 1192 | // every one of its instances' Region uniformly — spec's "one flag on the |
| 1193 | // app, all its instances follow" (appPlacement, fleets.go) — so when a |
| 1194 | // placement exists it OVERRIDES whatever the request body sent, and the |
| 1195 | // ordinary region/allow-list validation below never runs at all: a |
| 1196 | // "fleet/<name>" value would fail provision.ValidRegion's shape anyway |
| 1197 | // (no "/"), which is exactly why this has to be a check made BEFORE |
| 1198 | // that validation, not alongside it. |
| 1199 | reg, err := openRegistry(s.cfg.Registry) |
| 1200 | if err != nil { |
| 1201 | log.Printf("console: api instances create %s/%s: open registry: %v", acct.Sqid, app, err) |
| 1202 | jsonError(w, http.StatusBadGateway, "could not check this app's placement") |
| 1203 | return |
| 1204 | } |
| 1205 | placed, err := s.appPlacement(reg, acct, app) |
| 1206 | if err != nil { |
| 1207 | log.Printf("console: api instances create %s/%s: check placement: %v", acct.Sqid, app, err) |
| 1208 | jsonError(w, http.StatusBadGateway, "could not check this app's placement") |
| 1209 | return |
| 1210 | } |
| 1211 | // The app's region is the default for a create that names none |
| 1212 | // (nearest-region spec §7, R2). It runs BEFORE every check an explicit |
| 1213 | // region meets, and never for a fleet-placed app, a pooled manifest, a |
| 1214 | // replica, or anything but a socket-backed instance. |
| 1215 | raw := strings.TrimSpace(req.Region) |
| 1216 | regionFrom := "" |
| 1217 | if raw != "" { |
| 1218 | regionFrom = "request" |
| 1219 | } |
| 1220 | if raw == "" && placed == "" && m.Pool == "" && strings.TrimSpace(req.Role) == "" && |
| 1221 | (provision.Record{Kind: strings.TrimSpace(req.Kind), Addrs: req.Addrs}).DerivesPaths() { |
| 1222 | appRegion, _, err := s.appRegion(acct.ID, app) |
| 1223 | if err != nil { |
| 1224 | log.Printf("console: api instances create %s/%s: read app region: %v", acct.Sqid, app, err) |
| 1225 | jsonError(w, http.StatusBadGateway, msgAppRegionUnreadable) |
| 1226 | return |
| 1227 | } |
| 1228 | if appRegion != "" { |
| 1229 | raw, regionFrom = appRegion, "app" |
| 1230 | } |
| 1231 | } |
| 1232 | if placed != "" { |
| 1233 | regionFrom = "fleet" |
| 1234 | } |
| 1235 | // The home region's NAME normalizes to the record's "" before any |
| 1236 | // validation (issue #346): pickers speak the code, records keep the |
| 1237 | // unconstrained value reconcile has always scoped by. |
| 1238 | region := s.homeRegionIn(raw) |
| 1239 | // An APP-DERIVED region needs positive eligibility — the secrets key |
| 1240 | // included (R9) — and never rides the dormant gate (codex R2-2). |
| 1241 | if regionFrom == "app" { |
| 1242 | snap, err := s.liveEdges(ctx) |
| 1243 | if err != nil { |
| 1244 | log.Printf("console: api instances create %s/%s: read edges: %v", acct.Sqid, app, err) |
| 1245 | jsonError(w, http.StatusServiceUnavailable, msgAppRegionUnconfirmed(s.homeRegionOut(region))) |
| 1246 | return |
| 1247 | } |
| 1248 | // Incomplete evidence for a non-home region is "could not confirm" |
| 1249 | // (C28, the same answer as a failed listing), not C16: the region |
| 1250 | // may well be fine, and a box that could not be read is the one |
| 1251 | // that may not be (re-review r1). The home needs no evidence. |
| 1252 | if region != "" && !snap.EvidenceComplete() { |
| 1253 | log.Printf("console: api instances create %s/%s: incomplete edges or snapshot read, region %s unconfirmed", acct.Sqid, app, region) |
| 1254 | jsonError(w, http.StatusServiceUnavailable, msgAppRegionUnconfirmed(s.homeRegionOut(region))) |
| 1255 | return |
| 1256 | } |
| 1257 | cands := s.regionCands(snap) |
| 1258 | reason := "absent" |
| 1259 | if c, ok := findCandidate(cands, region); ok { |
| 1260 | reason = c.unavailableReason() |
| 1261 | } |
| 1262 | // Region HA (spec 2 §7, H27): the region is judged by the eligible |
| 1263 | // set the create door's chooser will choose from, never by every |
| 1264 | // sibling's own answer, and an empty set is the door's no-room |
| 1265 | // refusal, alert included. |
| 1266 | if x := s.placementChooser(""); x != nil { |
| 1267 | switch boxes, err := x.PlaceableBoxes(ctx, s.cfg.claimPlacementRegion(region)); { |
| 1268 | case errors.Is(err, hostops.ErrContract): |
| 1269 | reason = "absent" |
| 1270 | case err != nil: |
| 1271 | log.Printf("console: api instances create %s/%s: region %s placeable boxes: %v", acct.Sqid, app, region, err) |
| 1272 | jsonError(w, http.StatusServiceUnavailable, msgAppRegionUnconfirmed(s.homeRegionOut(region))) |
| 1273 | return |
| 1274 | case len(boxes) == 0: |
| 1275 | writeClaimResponseError(w, s.noRoom(ctx, region)) |
| 1276 | return |
| 1277 | default: |
| 1278 | reason = "" |
| 1279 | } |
| 1280 | } |
| 1281 | // Each list names only what an explicit request for this same |
| 1282 | // instance would pass (codex plan-R4 2): instanceChoices. |
| 1283 | if reason != "" { |
| 1284 | jsonError(w, http.StatusBadRequest, msgAppRegionUnavailable(s.homeRegionOut(region), reason, s.instanceChoices(ctx, m.Regions, cands, region))) |
| 1285 | return |
| 1286 | } |
| 1287 | if !instanceRegionAllowed(m.Regions, region) { |
| 1288 | jsonError(w, http.StatusBadRequest, msgAppRegionNotAllowed(s.homeRegionOut(region), s.instanceChoices(ctx, m.Regions, cands, region))) |
| 1289 | return |
| 1290 | } |
| 1291 | for _, rep := range req.Replicas { |
| 1292 | if s.homeRegionIn(strings.TrimSpace(rep)) == region { |
| 1293 | jsonError(w, http.StatusBadRequest, msgReplicaOverlap(s.homeRegionOut(region))) |
| 1294 | return |
| 1295 | } |
| 1296 | } |
| 1297 | } |
| 1298 | // The replica rails (#162) run BEFORE the region allow-list, because |
| 1299 | // half a decision is a different refusal from "wrong region": a role |
| 1300 | // with no placement, or a set on a replica, is a shape error the |
| 1301 | // caller can fix from the message alone, and the record must not |
| 1302 | // carry either into the shape validation below. The rails also |
| 1303 | // canonicalize the placement set, so the record ValidateShape sees — |
| 1304 | // and the bucket receives — is the canonical form. |
| 1305 | probeRec := provision.Record{Kind: strings.TrimSpace(req.Kind), Region: region, Role: strings.TrimSpace(req.Role)} |
| 1306 | if ref := checkReplicaCreate(req.replicaFields, &probeRec); ref != nil { |
| 1307 | jsonError(w, ref.Status, ref.Msg) |
| 1308 | return |
| 1309 | } |
| 1310 | if placed != "" { |
| 1311 | // A fleet-placed app governs every instance's Region uniformly, and |
| 1312 | // the armed/hosting gate below is skipped for it — the fleet is the |
| 1313 | // placement, validated by its own rail. A replica under that regime |
| 1314 | // would land wherever the fleet says, unchecked by anything that |
| 1315 | // knows what a replica placement means (#162): refused, not guessed. |
| 1316 | // Delete the app's placement first, or place the replica's account |
| 1317 | // on a fleet that spans the placement you want. |
| 1318 | if probeRec.Role == "replica" { |
| 1319 | jsonError(w, http.StatusBadRequest, |
| 1320 | "role replica on a fleet-placed app: the app's fleet placement ("+placed+") governs every instance's region, and a replica's placement must be an armed, hosting platform region — remove the app's placement first") |
| 1321 | return |
| 1322 | } |
| 1323 | region = placed |
| 1324 | } else if !provision.ValidRegion(region) || !instanceRegionAllowed(m.Regions, region) { |
| 1325 | msg := "invalid region" |
| 1326 | if len(m.Regions) > 0 { |
| 1327 | msg = "region " + region + " is not one this app runs in — allowed: " + strings.Join(m.Regions, ", ") |
| 1328 | } |
| 1329 | jsonError(w, http.StatusBadRequest, msg) |
| 1330 | return |
| 1331 | } |
| 1332 | // The armed-placement gate (issue #329): a non-empty region that names |
| 1333 | // no armed box is an instance NOTHING will ever mint — the record would |
| 1334 | // sit dark forever, which is worse than a refusal that lists the real |
| 1335 | // choices. Placed apps skipped (their value is a fleet, validated by its |
| 1336 | // own rail above); the unconstrained "" region skipped (it means "the |
| 1337 | // deployment's home", which always exists); and the whole gate is |
| 1338 | // dormant until some box publishes a placement code, so a console rolled |
| 1339 | // ahead of its fleet changes nothing. |
| 1340 | // |
| 1341 | // The gate has TWO bars, and which one applies depends on what is being |
| 1342 | // created. Every kind needs the region to be REACHABLE — a box nothing |
| 1343 | // can dial homes nothing. Only a socket-backed instance additionally |
| 1344 | // needs the region to HOST, because only that kind is spawned by an |
| 1345 | // activator: a static route is served from bucket bytes by every armed |
| 1346 | // edge with no process at all, and an addr-backed service proxies to a |
| 1347 | // process the platform never starts. Gating those two on hosting would |
| 1348 | // refuse a perfectly serveable static site in a region for want of a |
| 1349 | // hibernation bucket it would never have used. The predicate is |
| 1350 | // provision.Record's own (DerivesPaths), asked of a probe record rather |
| 1351 | // than restated here, so it cannot drift from what reconcile will mint. |
| 1352 | if placed == "" && region != "" { |
| 1353 | // Region HA (spec 2 §7, H27): a region with no box that has room is |
| 1354 | // the no-room refusal, before this gate answers it as a 400. |
| 1355 | if err := s.roomIn(ctx, region, m.Pool); err != nil { |
| 1356 | writeClaimResponseError(w, err) |
| 1357 | return |
| 1358 | } |
| 1359 | probe := provision.Record{Kind: strings.TrimSpace(req.Kind), Addrs: req.Addrs, Role: probeRec.Role} |
| 1360 | caps, published := placements(ctx, s.cfg.Releases) |
| 1361 | // The replica placement rides the same gate as the record's own |
| 1362 | // region (#162): a replica named into a placement no box announces |
| 1363 | // is a mirror nothing will ever provision — an order ledger in a |
| 1364 | // region that does not exist. Dormant with the rest of the gate |
| 1365 | // until a box publishes a placement code. |
| 1366 | if published && probe.Role == "replica" { |
| 1367 | switch { |
| 1368 | case !caps[region].Reachable: |
| 1369 | jsonError(w, http.StatusBadRequest, |
| 1370 | "replica placement "+region+" is not armed in this deployment — armed: "+armedCodes(reachableSet(caps))) |
| 1371 | return |
| 1372 | case !caps[region].Hosting: |
| 1373 | jsonError(w, http.StatusBadRequest, |
| 1374 | "replica placement "+region+" is reachable but cannot host an instance — no box there has its hibernation activator configured. Regions that can: "+armedCodes(hostingSet(caps))) |
| 1375 | return |
| 1376 | } |
| 1377 | } |
| 1378 | switch { |
| 1379 | case !published: |
| 1380 | // Dormant: the fleet predates the placement field. |
| 1381 | case !caps[region].Reachable: |
| 1382 | jsonError(w, http.StatusBadRequest, |
| 1383 | "region "+region+" is not armed in this deployment — armed: "+armedCodes(reachableSet(caps))) |
| 1384 | return |
| 1385 | case probe.DerivesPaths() && !caps[region].Hosting: |
| 1386 | jsonError(w, http.StatusBadRequest, |
| 1387 | "region "+region+" is reachable but cannot host an instance — no box there has its hibernation activator configured. Regions that can: "+armedCodes(hostingSet(caps))) |
| 1388 | return |
| 1389 | } |
| 1390 | // R9: a socket-backed instance or a replica needs its config, |
| 1391 | // secrets and store binding rendered on the box, which only a box |
| 1392 | // holding the deployment's secrets key can do. Refused only when a |
| 1393 | // CURRENT writer proves the region keyless; unknown keeps today's |
| 1394 | // behaviour, so roll order never refuses a working region. |
| 1395 | if probe.DerivesPaths() || probe.Role == "replica" { |
| 1396 | if snap, err := s.liveEdges(ctx); err == nil { |
| 1397 | if c, ok := findCandidate(s.regionCands(snap), region); ok && c.NoKeyProven { |
| 1398 | jsonError(w, http.StatusBadRequest, msgNoSecretsKey(s.homeRegionOut(region))) |
| 1399 | return |
| 1400 | } |
| 1401 | } |
| 1402 | } |
| 1403 | } |
| 1404 | channel := strings.TrimSpace(req.Channel) |
| 1405 | if channel == "" { |
| 1406 | resolved, ref := s.instanceChannel(ctx, acct.Sqid, app, m) |
| 1407 | if ref != nil { |
| 1408 | jsonError(w, ref.Status, ref.Msg) |
| 1409 | return |
| 1410 | } |
| 1411 | channel = resolved |
| 1412 | } |
| 1413 | if !release.ValidChannel(channel) { |
| 1414 | jsonError(w, http.StatusBadRequest, "invalid channel name") |
| 1415 | return |
| 1416 | } |
| 1417 | if ref := s.staticHeadRefusal(ctx, acct.Sqid, app, channel, strings.TrimSpace(req.Kind), host); ref != nil { |
| 1418 | jsonError(w, ref.Status, ref.Msg) |
| 1419 | return |
| 1420 | } |
| 1421 | rec := provision.Record{ |
| 1422 | Schema: 1, |
| 1423 | Host: host, |
| 1424 | Slug: strings.TrimSuffix(host, "."+m.Domain), |
| 1425 | App: app, |
| 1426 | Account: acct.Sqid, |
| 1427 | Pool: m.Pool, |
| 1428 | Region: region, |
| 1429 | Channel: channel, |
| 1430 | // UTC, like every other timestamp written to the bucket: these |
| 1431 | // records are read on boxes in other regions. |
| 1432 | CreatedAt: time.Now().UTC(), |
| 1433 | CreatedBy: principal, |
| 1434 | Kind: strings.TrimSpace(req.Kind), |
| 1435 | Environment: strings.TrimSpace(req.Environment), |
| 1436 | Addrs: req.Addrs, |
| 1437 | Health: req.Health, |
| 1438 | } |
| 1439 | // The replica rails already ran above (before the region allow-list), |
| 1440 | // on a probe record; the canonicalized answers carry onto the real one |
| 1441 | // here. The role lands only as "replica" — checkReplicaCreate refused |
| 1442 | // everything else before this point. |
| 1443 | rec.Role = probeRec.Role |
| 1444 | rec.Replicas = probeRec.Replicas |
| 1445 | // A static platform host is HA by default. Static is the one serving |
| 1446 | // class every platform box materializes unconditionally |
| 1447 | // (provision.everywhereClass): no unit, no process, content read from |
| 1448 | // the bucket channel pointer, so a far edge serves it with no home box |
| 1449 | // and no backhaul. The content half of that is already true — every |
| 1450 | // armed edge holds the site and its account wildcard cert. What was |
| 1451 | // missing is DNS: with no opt-in the host resolves to one edge, so |
| 1452 | // several healthy boxes hold the site and every visitor is sent to one |
| 1453 | // of them. |
| 1454 | // |
| 1455 | // Scoped to the platform namespace deliberately. internal/steering's |
| 1456 | // steeredHosts only ever builds latency records for hosts under the |
| 1457 | // apps domain, so defaulting a custom domain's record to "latency" |
| 1458 | // would write an opt-in no pass will ever honour (#306's shape); a |
| 1459 | // custom domain reaches the edges by its owner's own DNS, and the |
| 1460 | // console's attach guidance already tells them to CNAME the steering |
| 1461 | // hostname. |
| 1462 | // |
| 1463 | // Scoped to static deliberately too: a home-boxed route reaches a far |
| 1464 | // edge by backhaul, a different trade with a different cost, not one to |
| 1465 | // make silently for every app on the deployment. |
| 1466 | // |
| 1467 | // And it honours v1's pool rail, the same one the PATCH door enforces |
| 1468 | // with a 409 (steering needs the shared pool, because a pool's own |
| 1469 | // edges are not the deployment's). A default that quietly wrote what |
| 1470 | // the update door refuses would be the worse kind of inconsistency: the |
| 1471 | // record would carry an opt-in nobody could have asked for through the |
| 1472 | // front door, and the converger would skip it anyway. |
| 1473 | // |
| 1474 | // This is a value at MINT, never a converge: `carlos steering --app x |
| 1475 | // --host y off` is the escape hatch, and nothing re-stamps a record |
| 1476 | // that already exists. The opt-in is inert on a deployment that has not |
| 1477 | // set CARLOS_STEERING_ZONE_ID, so a single-box deployment states the |
| 1478 | // intent harmlessly and becomes HA the day it arms a second edge. |
| 1479 | if rec.Kind == "static" && platformNamespace && rec.Pool == "" { |
| 1480 | rec.Steering = "latency" |
| 1481 | } |
| 1482 | // The app's private-storage policy (issue #412, spec §6): every instance |
| 1483 | // gets a config environment of its own, named for the host's first |
| 1484 | // label, with an object store bound to it. |
| 1485 | // |
| 1486 | // An EXPLICIT environment in the request wins, and the policy still |
| 1487 | // applies to it. The policy fills a blank; it does not seize the axis. |
| 1488 | // What it promises is that this record's bundle carries the store |
| 1489 | // coordinates — not that the bundle has any particular name — so |
| 1490 | // honouring the caller's own name keeps `carlos canary` working on such |
| 1491 | // an app, and two instances naming one environment share its store |
| 1492 | // exactly as they already share its config (#408 §5: sharing is |
| 1493 | // visible, not refused). |
| 1494 | // |
| 1495 | // Stamped BEFORE ValidateShape so the shape rail sees the same |
| 1496 | // environment the box will, and before the ceiling and the walk below so |
| 1497 | // neither can be asked about a record that was never legal. |
| 1498 | // |
| 1499 | // A seeded instance takes no explicit environment under the policy: its |
| 1500 | // store must be new and empty, and a named one may be shared (spec §4). |
| 1501 | if seeded && m.PrivateStorage && strings.TrimSpace(req.Environment) != "" { |
| 1502 | s.refuseSeededEnvironment(w) |
| 1503 | return |
| 1504 | } |
| 1505 | if m.PrivateStorage { |
| 1506 | env, derived := rec.Environment, false |
| 1507 | if env == "" { |
| 1508 | env, derived = rec.Slug, true |
| 1509 | } |
| 1510 | if ref := s.instanceStoreNames(acct, app, env, derived); ref != nil { |
| 1511 | jsonError(w, ref.Status, ref.Msg) |
| 1512 | return |
| 1513 | } |
| 1514 | rec.Environment = env |
| 1515 | } |
| 1516 | // The shape rails, at the door — kind vocabulary, one upstream shape per |
| 1517 | // record, addr syntax, and the combinations that would wedge a host (a |
| 1518 | // static route with an upstream, a hibernating addr-backed one). The |
| 1519 | // message is ValidateShape's own: it names WHICH upstream is wrong, which |
| 1520 | // is the only thing the caller can act on, and it is the identical |
| 1521 | // sentence the box's reconcile pass would print if a record carrying this |
| 1522 | // shape ever reached it from another writer. |
| 1523 | if err := provision.ValidateShape(rec); err != nil { |
| 1524 | jsonError(w, http.StatusBadRequest, err.Error()) |
| 1525 | return |
| 1526 | } |
| 1527 | if s.refuseMemberUpstreams(w, p, rec.Addrs, nil) { |
| 1528 | return |
| 1529 | } |
| 1530 | // THE STORE IS PROVISIONED BEFORE THE RECORD IS WRITTEN, and a failure |
| 1531 | // here means no record at all (issue #412, spec §7). This ordering is |
| 1532 | // the load-bearing decision of the feature, so it is argued here rather |
| 1533 | // than assumed: |
| 1534 | // |
| 1535 | // Record first, and a failed walk leaves an instance whose bundle has no |
| 1536 | // store coordinates. It wakes, finds none, and runs on local disk — the |
| 1537 | // exact outcome the policy was declared to prevent, arrived at silently, |
| 1538 | // with no verb to repair it (there is no `instances create --resume`). |
| 1539 | // |
| 1540 | // Store first, and a failed record write leaves a store for an |
| 1541 | // environment no instance is bound to. That is visible — `store status` |
| 1542 | // lists every store of the app — and, because the environment is DERIVED |
| 1543 | // FROM THE HOST, a retry of the same host reuses it exactly: GrantIn is |
| 1544 | // idempotent, so the second attempt short-circuits and costs nothing. |
| 1545 | // |
| 1546 | // So: no instance record is ever created under this policy without its |
| 1547 | // store already provisioned and delivered. |
| 1548 | privateBucket := "" |
| 1549 | if m.PrivateStorage { |
| 1550 | if ref := s.storeCeiling(ctx, acct, app, rec.Environment); ref != nil { |
| 1551 | jsonError(w, ref.Status, ref.Msg) |
| 1552 | return |
| 1553 | } |
| 1554 | } |
| 1555 | // The leftover-replica rail (#199, createleftovers.go): a host still |
| 1556 | // holding a deleted instance's replica or move staging is refused, so |
| 1557 | // the new record cannot inherit that database. Last of the checks, after |
| 1558 | // every validation above, and BEFORE the store walk below, so a refused |
| 1559 | // create never provisions a bucket either. |
| 1560 | if ref := s.leftoverReplicaRefusal(r.Context(), leftoverCreate{ |
| 1561 | logTag: "api instances create " + acct.Sqid + "/" + app, |
| 1562 | host: host, region: region, pool: rec.Pool, |
| 1563 | exists: func(ctx context.Context) (bool, error) { |
| 1564 | _, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host) |
| 1565 | if errors.Is(err, os.ErrNotExist) { |
| 1566 | return false, nil |
| 1567 | } |
| 1568 | return err == nil, err |
| 1569 | }, |
| 1570 | // The same two answers the CreateRecord call below gives. |
| 1571 | duplicate: doorRefusal{http.StatusConflict, "instance " + host + " already exists"}, |
| 1572 | readFailed: doorRefusal{http.StatusBadGateway, "could not create the instance — try again"}, |
| 1573 | taken: host + " still holds data from a deleted instance. Choose another address, or ask an operator to purge it.", |
| 1574 | unsure: "could not check whether " + host + " is free. Try again.", |
| 1575 | }); ref != nil { |
| 1576 | jsonError(w, ref.Status, ref.Msg) |
| 1577 | return |
| 1578 | } |
| 1579 | // Plans 3a: an instance is a running thing. Checked once here, so plain, |
| 1580 | // seeded and imported creates all meet the cap, under the account lock. |
| 1581 | defer s.planLock(acct)() |
| 1582 | if err := s.checkRunningThings(r.Context(), acct, 1); err != nil { |
| 1583 | if isPlanCap(err) { |
| 1584 | writeJSON(w, http.StatusConflict, map[string]any{"error": err.Error(), "code": "cap"}) |
| 1585 | return |
| 1586 | } |
| 1587 | log.Printf("console: api instances create %s/%s: plan cap: %v", acct.Sqid, app, err) |
| 1588 | jsonError(w, http.StatusBadGateway, "could not create the instance — try again") |
| 1589 | return |
| 1590 | } |
| 1591 | // Every ordinary check has passed: a seed door takes over from here, and |
| 1592 | // writes its own record (apiinstanceseed.go). |
| 1593 | if req.From != nil { |
| 1594 | s.createFrom(w, r, seedCreate{acct: acct, app: app, principal: principal, message: message, m: m, |
| 1595 | rec: rec, placed: placed, regionFrom: regionFrom, siblingLabel: siblingLabel}, *req.From) |
| 1596 | return |
| 1597 | } |
| 1598 | if req.Import != nil { |
| 1599 | s.createImport(w, r, seedCreate{acct: acct, app: app, principal: principal, message: message, m: m, |
| 1600 | rec: rec, placed: placed, regionFrom: regionFrom, siblingLabel: siblingLabel}, *req.Import) |
| 1601 | return |
| 1602 | } |
| 1603 | // A fresh budget for the write and its rollback: validation and the |
| 1604 | // rail's listings may have spent most of the original one. |
| 1605 | cancel() |
| 1606 | ctx, cancel = context.WithTimeout(r.Context(), instanceStoreTimeout) |
| 1607 | defer cancel() |
| 1608 | buildClaim := func(now time.Time, acquisition string) (hostops.Record, error) { |
| 1609 | rec.CreatedAt = time.Time{} |
| 1610 | rec.Acquisition = acquisition |
| 1611 | region := s.cfg.claimPlacementRegion(rec.Region) |
| 1612 | prepared, err := hostops.PrepareInstance(rec, now, region) |
| 1613 | if err == nil { |
| 1614 | rec.CreatedAt = now |
| 1615 | } |
| 1616 | return prepared, err |
| 1617 | } |
| 1618 | // Region HA (spec 2 §7, H27): the box is chosen before the door's first |
| 1619 | // write, the environment binding below, so a create refused for want of |
| 1620 | // room changes nothing. beginClaimCreate keeps this choice. |
| 1621 | createContext := r.Context() |
| 1622 | if s.cfg.HostClaim { |
| 1623 | var chooseErr error |
| 1624 | if createContext, chooseErr = s.chooseCreateFirst(createContext, host, buildClaim); chooseErr != nil { |
| 1625 | writeClaimResponseError(w, chooseErr) |
| 1626 | return |
| 1627 | } |
| 1628 | } |
| 1629 | // An environment named without a store walk of its own takes the store |
| 1630 | // from a seed that made it (final review F3, seedenvbind.go). |
| 1631 | if !m.PrivateStorage && !s.bindEnvironmentCreate(ctx, w, rec) { |
| 1632 | return |
| 1633 | } |
| 1634 | var claimRequest *claimCreateRequest |
| 1635 | if s.cfg.HostClaim { |
| 1636 | var createErr error |
| 1637 | claimRequest, createErr = s.beginClaimCreate(createContext, acct, app, host, principal, buildClaim) |
| 1638 | if createErr != nil { |
| 1639 | writeClaimResponseError(w, createErr) |
| 1640 | return |
| 1641 | } |
| 1642 | defer claimRequest.Close() |
| 1643 | createContext = claimRequest.Context |
| 1644 | } |
| 1645 | if m.PrivateStorage { |
| 1646 | // r.Context(), not ctx: the walk takes storeGrantTimeout, which is |
| 1647 | // longer than this handler's own deadline. |
| 1648 | var storeResponse http.ResponseWriter = w |
| 1649 | if claimRequest != nil { |
| 1650 | storeResponse = &claimPrivateResponse{header: make(http.Header)} |
| 1651 | } |
| 1652 | storeRec, ok := s.instanceStoreProvision(createContext, storeResponse, acct, app, rec.Environment, principal) |
| 1653 | if !ok { |
| 1654 | if claimRequest != nil { |
| 1655 | writeJSON(w, http.StatusAccepted, map[string]string{"host": host, "operation_id": claimRequest.Observed.Claim.Op.ID, "phase": claimRequest.Observed.Claim.Op.Kind}) |
| 1656 | } |
| 1657 | return |
| 1658 | } |
| 1659 | privateBucket = storeBucketName(storeRec) |
| 1660 | // And a fresh deadline for what is left. The walk can have consumed |
| 1661 | // every second of the handler's original budget, and the record |
| 1662 | // write below inheriting a spent context would fail immediately |
| 1663 | // after the bucket had already been paid for — the one shape of |
| 1664 | // failure this whole ordering exists to avoid. |
| 1665 | cancel() |
| 1666 | var recancel context.CancelFunc |
| 1667 | ctx, recancel = context.WithTimeout(r.Context(), instanceStoreTimeout) |
| 1668 | defer recancel() |
| 1669 | } |
| 1670 | var recordWriteErr error |
| 1671 | if claimRequest != nil { |
| 1672 | if !s.finishClaimCreate(w, claimRequest) { |
| 1673 | return |
| 1674 | } |
| 1675 | } else { |
| 1676 | recordWriteErr = provision.CreateRecord(ctx, s.cfg.Releases, rec) |
| 1677 | } |
| 1678 | if err := recordWriteErr; err != nil { |
| 1679 | if errors.Is(err, release.ErrCASConflict) { |
| 1680 | // A lost race against a second signup for the same host, or a |
| 1681 | // plain duplicate. Either way the winner's record stands. |
| 1682 | jsonError(w, http.StatusConflict, "instance "+host+" already exists") |
| 1683 | return |
| 1684 | } |
| 1685 | log.Printf("console: api instances create %s/%s/%s: %v", acct.Sqid, app, host, err) |
| 1686 | jsonError(w, http.StatusBadGateway, "could not create the instance — try again") |
| 1687 | return |
| 1688 | } |
| 1689 | // The sibling check above and this write are two stores, so an app can be |
| 1690 | // created under the label between them (claimApp asked apexHolder before |
| 1691 | // the record existed). Re-read after writing, and on a hit take the record |
| 1692 | // back: the name that arrived owns its apex. The same after-the-commit |
| 1693 | // re-check a rename makes (appsettings.go). |
| 1694 | if siblingLabel != "" { |
| 1695 | if live, trashed, err := s.store.AppNameState(acct.ID, siblingLabel); err == nil && (live || trashed) { |
| 1696 | if derr := s.retireInstanceRecord(ctx, acct.Sqid, app, host, principal, mustCreateIncarnation(acct.Sqid, app, host, rec.CreatedAt)); derr != nil { |
| 1697 | log.Printf("console: api instances create %s/%s/%s: %s was claimed mid-create and UNDO FAILED: %v", acct.Sqid, app, host, siblingLabel, derr) |
| 1698 | } |
| 1699 | if privateBucket != "" { |
| 1700 | // The one window where this feature's ordering leaves |
| 1701 | // residue, logged in those words rather than papered over: |
| 1702 | // tearing the bucket down from a path nobody drove would |
| 1703 | // make this the platform's first automatic destroyer of |
| 1704 | // tenant data, which the per-environment store spec rules |
| 1705 | // out (#408 §3). It is listed by `carlos store status`. |
| 1706 | log.Printf("console: api instances create %s/%s/%s: %s was claimed mid-create; the record was withdrawn and its store %s WAS LEFT BEHIND", acct.Sqid, app, host, siblingLabel, privateBucket) |
| 1707 | } |
| 1708 | jsonError(w, http.StatusConflict, |
| 1709 | host+" is app "+siblingLabel+"'s platform apex — it was created while this instance was being written; the instance was not kept") |
| 1710 | return |
| 1711 | } |
| 1712 | } |
| 1713 | // DNS seam (spec §10): on a single-region deployment nothing writes |
| 1714 | // DNS — the app's wildcard already resolves every instance host to the |
| 1715 | // edge. Multi-region wants <slug>.<domain> CNAME -> <region>.<domain> |
| 1716 | // (TTL 60) written HERE, after the record write and before the |
| 1717 | // response, behind an interface the deployment configures (nil = |
| 1718 | // wildcard-covered). Recorded, not built. |
| 1719 | s.auditActivity(acct, app, principal, Activity{ |
| 1720 | Action: "instance-create", TargetType: "instance", Target: host, Message: message, |
| 1721 | Detail: "created instance " + host + " (cli)", |
| 1722 | }) |
| 1723 | // The record is what a box's reconcile leg mints a route from, so this is |
| 1724 | // a box-consumed write (carlos-deploy §3): announce it, and the instance |
| 1725 | // is serving seconds after signup instead of on the hour. |
| 1726 | s.bumpSerial(ctx) |
| 1727 | out := recordView(rec) |
| 1728 | out.PrivateStoreBucket = privateBucket |
| 1729 | out.RegionFrom = regionFrom |
| 1730 | // Every non-fleet placement is named, replicas included (codex |
| 1731 | // plan-R3 6); a fleet-placed record's Region is the fleet itself. |
| 1732 | if placed == "" { |
| 1733 | out.RegionName = s.placeOf(rec.Region, s.candsOrHome(ctx)) |
| 1734 | } |
| 1735 | writeJSON(w, http.StatusCreated, out) |
| 1736 | } |
| 1737 | |
| 1738 | // instanceChannel answers "which channel is this record born on" for a |
| 1739 | // create that named none — amadan carlos/platform#5, where a brand-new app's |
| 1740 | // first instance landed on "stable" and the first deploy then met stable's |
| 1741 | // passkey, as did the obvious repair of moving it to edge. |
| 1742 | // |
| 1743 | // The rungs, in order, and why each one is above the next: |
| 1744 | // |
| 1745 | // manifest channel app policy, written at `carlos instances enable |
| 1746 | // -channel`. An app that declared one is not re-decided |
| 1747 | // here, in either direction — this is what keeps tito's |
| 1748 | // per-customer creates landing where every sibling is. |
| 1749 | // sibling records the channel this app's OTHER instances follow, when |
| 1750 | // they agree. A record per customer is the flagship's |
| 1751 | // shape, and a new one quietly landing on the pipeline |
| 1752 | // entry would serve that customer the edge build. |
| 1753 | // canary/* siblings do not vote: a canary instance is |
| 1754 | // made to verify a build and deleted after, so the |
| 1755 | // throwaway must not set policy for the real one. |
| 1756 | // pipeline entry where this app's releases actually land |
| 1757 | // (release.Pipeline.Entry), which is what `carlos |
| 1758 | // deploy` resolves its own channel from. An app with no |
| 1759 | // pipeline.json falls to the starter pipeline's entry, |
| 1760 | // which is also the legacy ladder's: nothing is promoted |
| 1761 | // ONTO edge — merging is what puts a build there. |
| 1762 | // |
| 1763 | // Two cases refuse rather than guess, because a record's channel is chosen |
| 1764 | // once and this door cannot take it back: |
| 1765 | // |
| 1766 | // an unreadable pipeline is NOT an absent one (apipipeline.go's rule) — a |
| 1767 | // bucket hiccup must not mint a production route by falling through, and |
| 1768 | // instances that disagree have no default worth inventing (attachChannel's |
| 1769 | // refusal, same words, same reason). |
| 1770 | // |
| 1771 | // A caller who named a channel never reaches here: an explicit channel is |
| 1772 | // the caller's decision, is shape-checked only, and costs no bucket reads. |
| 1773 | func (s *Server) instanceChannel(ctx context.Context, sqid, app string, m *provision.Manifest) (string, *doorRefusal) { |
| 1774 | if m.Channel != "" { |
| 1775 | return m.Channel, nil |
| 1776 | } |
| 1777 | recs, _, err := provision.ListRecords(ctx, s.cfg.Releases, sqid, app, instanceListMax) |
| 1778 | if err != nil { |
| 1779 | log.Printf("console: api instances create %s/%s: list siblings: %v", sqid, app, err) |
| 1780 | return "", &doorRefusal{http.StatusBadGateway, |
| 1781 | "could not read this app's instances — try again, or pass -channel"} |
| 1782 | } |
| 1783 | var found []string |
| 1784 | for _, rec := range recs { |
| 1785 | if rung, ok := release.Rung(rec.Channel); rec.Channel == "" || (ok && rung == release.RungCanary) { |
| 1786 | continue |
| 1787 | } |
| 1788 | if !slices.Contains(found, rec.Channel) { |
| 1789 | found = append(found, rec.Channel) |
| 1790 | } |
| 1791 | } |
| 1792 | slices.Sort(found) |
| 1793 | if len(found) > 1 { |
| 1794 | return "", &doorRefusal{http.StatusUnprocessableEntity, |
| 1795 | "this app's instances follow several channels (" + strings.Join(found, ", ") + |
| 1796 | ") — pass -channel to say which one this instance follows"} |
| 1797 | } |
| 1798 | if len(found) == 1 { |
| 1799 | return found[0], nil |
| 1800 | } |
| 1801 | p, err := release.GetPipeline(ctx, s.cfg.Releases, s.cfg.ReleasePub, sqid, app) |
| 1802 | if err != nil { |
| 1803 | log.Printf("console: api instances create %s/%s: pipeline read: %v", sqid, app, err) |
| 1804 | return "", &doorRefusal{http.StatusBadGateway, |
| 1805 | "could not read this app's pipeline — try again, or pass -channel"} |
| 1806 | } |
| 1807 | if p != nil { |
| 1808 | return p.Entry(), nil |
| 1809 | } |
| 1810 | return release.DefaultPipelineChannels()[0].Name, nil |
| 1811 | } |
| 1812 | |
| 1813 | // staticHeadRefusal refuses the one instance record that can never work: a |
| 1814 | // process-backed route on a channel whose head is a STATIC release. |
| 1815 | // |
| 1816 | // The record would be created happily, provisioned into a real route, and |
| 1817 | // then fail every wake forever — a static release's manifest carries a "site" |
| 1818 | // tar and no build for any box's platform, so nothing is ever staged and no |
| 1819 | // live symlink is ever written. What the member saw before this door existed |
| 1820 | // was "nothing is promoted for this instance's channel" against a `carlos |
| 1821 | // channels` that showed the promote plainly there, with no path from either |
| 1822 | // to the real problem (skills#29). internal/release's noHeadReason now names |
| 1823 | // that state honestly when a box is already in it; this is the door that |
| 1824 | // stops it being reachable. |
| 1825 | // |
| 1826 | // Only the exec shapes are gated. A "service" record proxies to declared |
| 1827 | // upstreams and a "static" record reads the channel pointer directly — |
| 1828 | // neither stages a binary, so neither has anything to mismatch. An empty kind |
| 1829 | // is the default, which IS "instance": that is precisely the request the |
| 1830 | // reporter made. |
| 1831 | // |
| 1832 | // The check spends two bucket reads on a create, and only a create. Every |
| 1833 | // failure to read degrades to nil — this is a guard against a shape mistake, |
| 1834 | // not a rail against a hostile caller, and a bucket hiccup must not stop an |
| 1835 | // ordinary member declaring an ordinary instance. A brand-new app with |
| 1836 | // nothing promoted yet reads as nil too, which is the common first-instance |
| 1837 | // case and stays exactly as fast as it was. |
| 1838 | func (s *Server) staticHeadRefusal(ctx context.Context, sqid, app, channel, kind, host string) *doorRefusal { |
| 1839 | if kind != "" && kind != "instance" { |
| 1840 | return nil |
| 1841 | } |
| 1842 | ptr, err := release.Channel(ctx, s.cfg.Releases, s.cfg.ReleasePub, sqid, app, channel) |
| 1843 | if err != nil || ptr == nil || ptr.Version == "" { |
| 1844 | return nil |
| 1845 | } |
| 1846 | m, err := release.GetManifest(ctx, s.cfg.Releases, sqid, app, ptr.Version) |
| 1847 | if err != nil || m == nil || m.Kind != "static" { |
| 1848 | return nil |
| 1849 | } |
| 1850 | return &doorRefusal{http.StatusUnprocessableEntity, |
| 1851 | "channel " + channel + " serves a static release (" + ptr.Version + "), and an instance runs a process — " + |
| 1852 | "a static site is served straight off the channel pointer, so it needs no instance at all. " + |
| 1853 | "Pass --kind static, or let one command do the whole thing: carlos deploy --app " + app + |
| 1854 | " --kind static --host " + host + " <site-dir>"} |
| 1855 | } |
| 1856 | |
| 1857 | // refuseMemberUpstreams answers whether this caller may point a route at |
| 1858 | // addrs, and writes the refusal when not. Stating an upstream is an |
| 1859 | // operator's act: the edge dials these addresses from inside the platform, |
| 1860 | // so a member's could reach the box's own services (the console on |
| 1861 | // 127.0.0.1:8090) or another tenant's private upstream (discussion 214, |
| 1862 | // item 4). Paul's ruling on discussion 50 (option a): members |
| 1863 | // state none until the "own fleet" allowance exists, public addresses |
| 1864 | // included, and operators keep today's rail. |
| 1865 | // |
| 1866 | // kept lists the addresses the record already carries. Those pass whoever |
| 1867 | // sends them, so a member whose route an operator pointed somewhere can |
| 1868 | // keep it, change its probe, or drop a dead upstream; only adding one is |
| 1869 | // refused. |
| 1870 | // |
| 1871 | // "Operator" is a person holding the operator bit. A service credential |
| 1872 | // never is one, whichever account it belongs to. A failed operator lookup |
| 1873 | // refuses, so a database blip is not the way past it. |
| 1874 | func (s *Server) refuseMemberUpstreams(w http.ResponseWriter, p Principal, addrs, kept []string) bool { |
| 1875 | added := false |
| 1876 | for _, a := range addrs { |
| 1877 | if !slices.Contains(kept, a) { |
| 1878 | added = true |
| 1879 | break |
| 1880 | } |
| 1881 | } |
| 1882 | if !added { |
| 1883 | return false |
| 1884 | } |
| 1885 | if p.Kind == PrincipalUser { |
| 1886 | op, err := s.store.IsOperator(p.Address) |
| 1887 | if err != nil { |
| 1888 | log.Printf("console: upstream rail: is-operator %s: %v", p.Address, err) |
| 1889 | jsonError(w, http.StatusServiceUnavailable, "could not check who may set these upstreams — nothing was changed. Try again.") |
| 1890 | return true |
| 1891 | } |
| 1892 | if op { |
| 1893 | return false |
| 1894 | } |
| 1895 | } |
| 1896 | jsonError(w, http.StatusForbidden, "only a platform operator can point an instance at its own backends") |
| 1897 | return true |
| 1898 | } |
| 1899 | |
| 1900 | // handleAPIInstanceUpdate repoints a live instance's UPSTREAM SET, probe |
| 1901 | // config and (Task 3, multi-region-steering plan) DNS STEERING opt-in — the |
| 1902 | // fields of a record that are meant to change while the instance keeps |
| 1903 | // serving (HA design, issue #152; steering, 2026-08-20 multi-region ingress |
| 1904 | // design Part 4). Everything else about a record stays where it is: identity |
| 1905 | // (host/app/account) is checked at create and never touched again, policy |
| 1906 | // (pool, domain) is the manifest's, and region has its own writer in |
| 1907 | // placement. |
| 1908 | // |
| 1909 | // It is a PATCH, and the body's fields are independently optional, because |
| 1910 | // "leave the probe alone while I add an upstream" is the ordinary request. A |
| 1911 | // body carrying NONE of them is refused rather than treated as a no-op write: |
| 1912 | // it would cost a bucket round trip and a fleet-wide serial bump to change |
| 1913 | // nothing. |
| 1914 | // |
| 1915 | // The write is guarded by the etag the record was READ under |
| 1916 | // (provision.UpdateRecord): the caller decided this upstream set (or this |
| 1917 | // steering opt-in) given the record they were shown, so a concurrent writer |
| 1918 | // means the decision was made about bytes that no longer exist, and the |
| 1919 | // honest answer is 409 — the same answer the enable door gives for the same |
| 1920 | // reason. |
| 1921 | // |
| 1922 | // Steering carries its own two rails, beyond the CAS guard every field here |
| 1923 | // shares: an invalid value is ValidateShape's own 400 (ValidSteering is |
| 1924 | // already one of its checks, so a bad vocabulary word needs no extra code |
| 1925 | // here), and asking to turn latency ON for a record whose Pool != "" is a |
| 1926 | // 409 naming the v1 constraint verbatim (plan-2 ledger: a steered |
| 1927 | // pool-scoped route would get latency records pointing at edges that |
| 1928 | // structurally cannot serve it — the per-pool-record-scoping alternative is |
| 1929 | // a recorded future, not v1). The rail reads the FETCHED RECORD's Pool — |
| 1930 | // stamped once at create and never touched by this door — not the app's |
| 1931 | // manifest, which may have moved to a different pool since; the record is |
| 1932 | // the source of truth this door edits, and the manifest's current value is |
| 1933 | // not it. Turning steering OFF is never blocked by this rail: only the |
| 1934 | // latency-on request needs the shared pool. |
| 1935 | // |
| 1936 | // Every refusal before the principal is proven is the family's byte-identical |
| 1937 | // 404 (instanceGate), including a host that names no record: a record's |
| 1938 | // existence is not a thing to probe for here any more than it is on GET. |
| 1939 | func (s *Server) handleAPIInstanceUpdate(w http.ResponseWriter, r *http.Request) { |
| 1940 | principal, acct, app, ok := s.instanceGate(w, r) |
| 1941 | if !ok { |
| 1942 | return |
| 1943 | } |
| 1944 | // Same split as create, for a sharper reason: the provisioning |
| 1945 | // reconciler converges a route's Channel and Environment FROM this |
| 1946 | // record on every pass (AGENTS.md), so a PATCH here repoints the |
| 1947 | // control plane. An operator doing that deliberately is legitimate; a CI |
| 1948 | // credential doing it is the console being moved by something with no |
| 1949 | // standing to move it. Placed before the host shape check so that no |
| 1950 | // branch of this handler reaches a write without passing it. |
| 1951 | p, _ := principalFrom(r.Context()) |
| 1952 | if s.refuseServiceControlPlane(w, p, acct.ID, app, "change its instance records") { |
| 1953 | return |
| 1954 | } |
| 1955 | host := r.PathValue("host") |
| 1956 | // Shape-checked before the key builder, exactly as GET does it and for |
| 1957 | // the same reason: a {host} segment arrives percent-decoded, so |
| 1958 | // "..%2f..%2fchannels" reaches this handler as a path with separators in |
| 1959 | // it and RecordKey would happily join it. The manifest's suffix rule is |
| 1960 | // deliberately NOT applied — that is create-time policy ("which hosts may |
| 1961 | // this app claim"), and an app that has since moved domains must still be |
| 1962 | // able to repoint the upstreams of the instances it already has. |
| 1963 | if !provision.ValidHostShape(host) { |
| 1964 | http.NotFound(w, r) |
| 1965 | return |
| 1966 | } |
| 1967 | // Addrs is a POINTER and Probe a RawMessage because absent, empty and |
| 1968 | // null are three different requests (the apiConfigApply precedent): an |
| 1969 | // omitted field leaves what the record carries, `"addrs":[]` clears the |
| 1970 | // upstream set, and `"probe":null` clears the probe while `"probe":{}` |
| 1971 | // asks for the platform defaults. |
| 1972 | // |
| 1973 | // The two are deliberately NOT symmetric about null, and the asymmetry is |
| 1974 | // worth stating because it is invisible from the types: `"probe":null` |
| 1975 | // CLEARS the probe (a RawMessage records that the key was present), while |
| 1976 | // `"addrs":null` is indistinguishable from an absent key to a *[]string |
| 1977 | // and therefore means "leave the upstream set alone". Clearing the set is |
| 1978 | // spelled `"addrs":[]`, which is also the only spelling that says what it |
| 1979 | // means. |
| 1980 | // |
| 1981 | // Steering is a *string, plainer than either: it is one scalar fact, not |
| 1982 | // a set or a block, so there is no empty-vs-null distinction to carry. |
| 1983 | // Absent (nil) leaves the record's steering alone; `"steering":""` turns |
| 1984 | // it off; `"steering":"latency"` turns it on (subject to the pool rail |
| 1985 | // below). |
| 1986 | // Environment is a *string for the same reason Steering is: one scalar |
| 1987 | // fact, so absent (nil) leaves the record's binding alone and |
| 1988 | // `"environment":""` returns the route to the app's default bundle. |
| 1989 | // Rebinding is exactly what the handler doc above describes as |
| 1990 | // legitimate-when-deliberate — this door repoints the control plane, and |
| 1991 | // refuseServiceControlPlane has already decided whether this caller may. |
| 1992 | var req struct { |
| 1993 | Addrs *[]string `json:"addrs"` |
| 1994 | Probe json.RawMessage `json:"probe"` |
| 1995 | Steering *string `json:"steering"` |
| 1996 | Environment *string `json:"environment"` |
| 1997 | // Gate is a *string like Steering: absent leaves the gate alone, |
| 1998 | // "" opens the host, anything else is the new password — hashed |
| 1999 | // here, never stored or echoed (spec §5.2). The plaintext crosses |
| 2000 | // the wire once, over TLS, the same as a sign-in. |
| 2001 | Gate *string `json:"gate"` |
| 2002 | } |
| 2003 | if !readJSON(w, r, &req) { |
| 2004 | return |
| 2005 | } |
| 2006 | if req.Addrs == nil && req.Probe == nil && req.Steering == nil && req.Environment == nil && req.Gate == nil { |
| 2007 | jsonError(w, http.StatusBadRequest, |
| 2008 | "nothing to update — send addrs, probe, steering, environment, gate, or a combination") |
| 2009 | return |
| 2010 | } |
| 2011 | var probe *provision.Health |
| 2012 | if req.Probe != nil { |
| 2013 | if err := json.Unmarshal(req.Probe, &probe); err != nil { |
| 2014 | jsonError(w, http.StatusBadRequest, |
| 2015 | `invalid probe — want an object like {"path":"/healthz","status":200} or null to stop probing`) |
| 2016 | return |
| 2017 | } |
| 2018 | } |
| 2019 | ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| 2020 | defer cancel() |
| 2021 | rec, etag, err := provision.GetRecordETag(ctx, s.cfg.Releases, acct.Sqid, app, host) |
| 2022 | if errors.Is(err, os.ErrNotExist) { |
| 2023 | http.NotFound(w, r) |
| 2024 | return |
| 2025 | } |
| 2026 | if err != nil { |
| 2027 | log.Printf("console: api instances update read %s/%s/%s: %v", acct.Sqid, app, host, err) |
| 2028 | jsonError(w, http.StatusBadGateway, "could not read the instance") |
| 2029 | return |
| 2030 | } |
| 2031 | // The record as it stands, kept whole before anything below mutates it: |
| 2032 | // the shape rail past ValidateShape is a comparison BETWEEN the stored |
| 2033 | // route and the patched one, not a property of either alone. |
| 2034 | stored := *rec |
| 2035 | if req.Addrs != nil { |
| 2036 | // This door states the upstream set WHOLE, so it also retires the |
| 2037 | // legacy single-addr spelling of the same fact. Leaving Addr standing |
| 2038 | // would make every patch of a record minted before Addrs existed fail |
| 2039 | // provision.ValidateShape's "addr and addrs both set" rail — a refusal |
| 2040 | // about a field the caller never sent, on the exact records (kass, |
| 2041 | // keymail, the consoles) this feature exists to give a second upstream |
| 2042 | // to. Clearing it is not a widening: AddrsOrDefault already folds Addr |
| 2043 | // into the same one-element set this replaces. |
| 2044 | rec.Addrs, rec.Addr = *req.Addrs, "" |
| 2045 | } |
| 2046 | if req.Probe != nil { |
| 2047 | rec.Health = probe |
| 2048 | } |
| 2049 | if req.Steering != nil { |
| 2050 | rec.Steering = *req.Steering |
| 2051 | } |
| 2052 | if req.Environment != nil { |
| 2053 | rec.Environment = strings.TrimSpace(*req.Environment) |
| 2054 | } |
| 2055 | // The gate is the one field this door does not store as it was sent: the |
| 2056 | // record carries a bcrypt hash and nothing else, so the plaintext lives |
| 2057 | // only in this stack frame. Not trimmed either — a password's leading and |
| 2058 | // trailing spaces are part of it, and the login POST compares the bytes |
| 2059 | // the member typed. |
| 2060 | if req.Gate != nil { |
| 2061 | if *req.Gate == "" { |
| 2062 | rec.Gate = "" |
| 2063 | } else { |
| 2064 | // CheckPassword's sentences name the rule and never the value: |
| 2065 | // nothing below (and nothing logged) can echo the password back. |
| 2066 | if err := gate.CheckPassword(*req.Gate); err != nil { |
| 2067 | jsonError(w, http.StatusBadRequest, err.Error()) |
| 2068 | return |
| 2069 | } |
| 2070 | h, err := gate.Hash(*req.Gate) |
| 2071 | if err != nil { |
| 2072 | log.Printf("console: api instances update %s/%s/%s: hash gate: %v", acct.Sqid, app, host, err) |
| 2073 | jsonError(w, http.StatusInternalServerError, "could not set the password — try again") |
| 2074 | return |
| 2075 | } |
| 2076 | rec.Gate = h |
| 2077 | } |
| 2078 | } |
| 2079 | // The same door the create path and the box's reconcile pass check |
| 2080 | // against, so a shape that could not have been created cannot be patched |
| 2081 | // into existence either — and the caller gets ValidateShape's own |
| 2082 | // sentence, which names the offending upstream. It is also this |
| 2083 | // door's own invalid-value 400 for steering: ValidSteering is already |
| 2084 | // one of ValidateShape's checks, so "geo" or "Latency" is refused here |
| 2085 | // with no code of this handler's own. |
| 2086 | if err := provision.ValidateShape(*rec); err != nil { |
| 2087 | jsonError(w, http.StatusBadRequest, err.Error()) |
| 2088 | return |
| 2089 | } |
| 2090 | // An address the record already carries is not a new statement, so a |
| 2091 | // member may keep it: dropping one dead upstream from an operator-made |
| 2092 | // set is theirs to do. |
| 2093 | if req.Addrs != nil && s.refuseMemberUpstreams(w, p, *req.Addrs, stored.AddrsOrDefault()) { |
| 2094 | return |
| 2095 | } |
| 2096 | // v1's pool guardrail (multi-region-steering plan, plan-2 ledger): |
| 2097 | // latency steering needs the SHARED pool, because a pool's own edges are |
| 2098 | // the only ones that can serve a pool-scoped route — a steered |
| 2099 | // non-shared-pool route would advertise latency records pointing at |
| 2100 | // edges that structurally cannot answer for it. The per-pool-record |
| 2101 | // alternative is a recorded future, not v1. |
| 2102 | // |
| 2103 | // rec.Pool is read straight off the bytes GetRecordETag fetched moments |
| 2104 | // ago — this door never writes Pool and never re-derives it from the |
| 2105 | // app's manifest, which may have moved to a different pool since the |
| 2106 | // record was created. The record is the source of truth this door |
| 2107 | // edits; the manifest's CURRENT value is not it, and using it here would |
| 2108 | // let an operator's later `instances enable` silently flip what an |
| 2109 | // already-provisioned route is allowed to do. Turning steering OFF is |
| 2110 | // never blocked: only a request that leaves this record at "latency" |
| 2111 | // needs the shared pool. |
| 2112 | if rec.Steering == "latency" && rec.Pool != "" { |
| 2113 | jsonError(w, http.StatusConflict, |
| 2114 | "steering requires the shared pool — pool-scoped steering is a recorded future") |
| 2115 | return |
| 2116 | } |
| 2117 | // The rail this door needs beyond ValidateShape: a patch may repoint a |
| 2118 | // route's upstreams, never change what KIND of thing the route is. |
| 2119 | // DerivesPaths is that line exactly — it decides whether the route owns a |
| 2120 | // <host>.sock and a <host>.db — and the reconcile pass refuses to converge |
| 2121 | // a row across it in place (the "record describes a different route SHAPE" |
| 2122 | // log). Allowing the patch anyway would answer 200 for a change that can |
| 2123 | // never take effect: a socket-backed instance handed addrs would keep |
| 2124 | // serving its socket while the box logged the same complaint every pass, |
| 2125 | // and an addrs-backed route emptied to nothing would become a route that |
| 2126 | // serves nothing at all. Transitions go through delete + create, which |
| 2127 | // this API already supports, and which lets the box retire the old route's |
| 2128 | // derived paths honestly. |
| 2129 | if stored.DerivesPaths() != rec.DerivesPaths() { |
| 2130 | jsonError(w, http.StatusBadRequest, |
| 2131 | "that changes the instance's route SHAPE, not just its upstreams — a socket-backed instance "+ |
| 2132 | "and an upstream-backed one are different routes, and a box cannot convert one into the other "+ |
| 2133 | "in place; delete "+host+" and create it again with the shape you want") |
| 2134 | return |
| 2135 | } |
| 2136 | // A new environment binding is serialised with a seed's store teardown, |
| 2137 | // and takes the store from the seed (final review F3, seedenvbind.go). |
| 2138 | unlockEnv, ok := s.lockEnvironmentUpdate(ctx, w, stored, *rec) |
| 2139 | if !ok { |
| 2140 | return |
| 2141 | } |
| 2142 | defer unlockEnv() |
| 2143 | // provision.UpdateRecord marshals the whole record through the canonical |
| 2144 | // writer: nothing here hand-builds an object body, so upstream ORDER — |
| 2145 | // which the reconcile pass's shapeMismatch compares verbatim against the |
| 2146 | // registry's joined column — lands exactly as the caller stated it. |
| 2147 | if err := provision.UpdateRecord(ctx, s.cfg.Releases, *rec, etag); err != nil { |
| 2148 | if errors.Is(err, release.ErrCASConflict) { |
| 2149 | jsonError(w, http.StatusConflict, |
| 2150 | "instance "+host+" changed while this request was in flight — read it again and re-apply") |
| 2151 | return |
| 2152 | } |
| 2153 | log.Printf("console: api instances update %s/%s/%s: %v", acct.Sqid, app, host, err) |
| 2154 | jsonError(w, http.StatusBadGateway, "could not update the instance — try again") |
| 2155 | return |
| 2156 | } |
| 2157 | // "upstreams" no longer names the whole of what this door can change |
| 2158 | // (steering joined it in Task 3), so the audit line names exactly the |
| 2159 | // fields THIS request carried rather than either overclaiming ("upstreams" |
| 2160 | // on a steering-only patch) or going field-agnostic ("updated instance |
| 2161 | // <host> (cli)" alone) — this is a security surface, and "did anyone turn |
| 2162 | // on steering for this host last week?" has to stay answerable from the |
| 2163 | // audit log. Built from the same req.* nil checks the mutation above |
| 2164 | // already made, so it can never drift from what was actually applied. |
| 2165 | var changed []string |
| 2166 | if req.Addrs != nil { |
| 2167 | changed = append(changed, "addrs") |
| 2168 | } |
| 2169 | if req.Probe != nil { |
| 2170 | changed = append(changed, "probe") |
| 2171 | } |
| 2172 | if req.Steering != nil { |
| 2173 | changed = append(changed, "steering") |
| 2174 | } |
| 2175 | if req.Environment != nil { |
| 2176 | changed = append(changed, "environment") |
| 2177 | } |
| 2178 | // "gate" names that the host's password was set or cleared, never which |
| 2179 | // it was: the audit log answers "did anyone change the gate on this host |
| 2180 | // last week?" without becoming a place a password can be read. |
| 2181 | if req.Gate != nil { |
| 2182 | changed = append(changed, "gate") |
| 2183 | } |
| 2184 | s.audit(acct, app, principal, "instance-update", |
| 2185 | "updated instance "+host+" ("+strings.Join(changed, ", ")+") (cli)") |
| 2186 | // Strictly after the write landed, and exactly once: the record is what a |
| 2187 | // box's reconcile leg converges upstreams from (provision.convergeOwned), |
| 2188 | // so this announcement is what makes the new set live in seconds instead |
| 2189 | // of on the hour. |
| 2190 | s.bumpSerial(ctx) |
| 2191 | writeJSON(w, http.StatusOK, recordView(*rec)) |
| 2192 | } |
| 2193 | |
| 2194 | // handleAPIInstanceDelete retires an instance's route authorization. The |
| 2195 | // record is deleted; the parked database and blobs are deliberately |
| 2196 | // untouched (spec §3.2) — retiring a route is not destroying a customer's |
| 2197 | // data. Idempotent: the caller's goal is "not there", and a second delete |
| 2198 | // has reached it. |
| 2199 | func (s *Server) handleAPIInstanceDelete(w http.ResponseWriter, r *http.Request) { |
| 2200 | principal, acct, app, ok := s.instanceGate(w, r) |
| 2201 | if !ok { |
| 2202 | return |
| 2203 | } |
| 2204 | message, ok := checkedActivityMessage(w, r.Header.Get(activityMessageHeader)) |
| 2205 | if !ok { |
| 2206 | return |
| 2207 | } |
| 2208 | // The eighth destructive door, and the one with the shortest fuse: |
| 2209 | // deleting a control-plane app's record is how the provisioning |
| 2210 | // reconciler's removal phase is TOLD to retire the console's route, so a |
| 2211 | // single call here takes the control plane off the air. That is the same |
| 2212 | // class of act as trash, purge and store-destroy, and it wears their |
| 2213 | // refusal — outright, for anyone, service or person alike, until the |
| 2214 | // flag is cleared (controlplane.go). |
| 2215 | // |
| 2216 | // FIRST, ahead of the shape check and the store: the app is already |
| 2217 | // past instanceGate's byte-identical 404, and nothing below this line |
| 2218 | // may run for a flagged app. |
| 2219 | if s.refuseControlPlane(w, acct, app, "deleted") { |
| 2220 | return |
| 2221 | } |
| 2222 | host := r.PathValue("host") |
| 2223 | ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| 2224 | defer cancel() |
| 2225 | // SHAPE ONLY, always — and the manifest is not read here at all. The |
| 2226 | // domain-suffix rule is CREATE-time policy ("which hosts may this app |
| 2227 | // claim"), and deletion has to outlive every version of it: a retired |
| 2228 | // manifest, an app that moved domains, a record written under a domain |
| 2229 | // the manifest no longer names. Any of those checked against today's |
| 2230 | // manifest would refuse to delete records that exist, stranding |
| 2231 | // provisioned routes with nothing left able to retire them. What |
| 2232 | // deletion actually needs from validation is the traversal defence: a |
| 2233 | // {host} arrives percent-decoded, so "..%2f..%2finstances.json" reaches |
| 2234 | // here with separators in it and RecordKey would happily join it. |
| 2235 | // ValidHostShape is exactly that check, and there is no path through |
| 2236 | // this handler on which an unvalidated host reaches RecordKey. |
| 2237 | if !provision.ValidHostShape(host) { |
| 2238 | jsonError(w, http.StatusBadRequest, "invalid host") |
| 2239 | return |
| 2240 | } |
| 2241 | // With CARLOS_MOVES_PHASE1 on, a host being moved is not deleted |
| 2242 | // (M-C19, apiinstancemove.go); with it off this is main's door. |
| 2243 | if s.refuseDeleteDuringMove(ctx, w, acct.Sqid, app, host) { |
| 2244 | return |
| 2245 | } |
| 2246 | if answered, finished := s.refuseImportDelete(w, r, acct, app, host); answered { |
| 2247 | if finished { |
| 2248 | s.auditActivity(acct, app, principal, Activity{ |
| 2249 | Action: "instance-delete", TargetType: "instance", Target: host, Message: message, |
| 2250 | Detail: "deleted instance " + host + " (cli)", |
| 2251 | }) |
| 2252 | s.bumpSerial(ctx) |
| 2253 | writeJSON(w, http.StatusOK, map[string]any{"result": "deleted"}) |
| 2254 | } |
| 2255 | return |
| 2256 | } |
| 2257 | if err := s.retireInstanceRecord(ctx, acct.Sqid, app, host, principal, ""); err != nil { |
| 2258 | // An accepted delete is this principal's durable request: audit it |
| 2259 | // and announce it before answering 202 with the operation. |
| 2260 | if accepted, ok := claimRetireAcceptedOf(err); ok { |
| 2261 | s.auditActivity(acct, app, principal, Activity{ |
| 2262 | Action: "instance-delete", TargetType: "instance", Target: host, Message: message, |
| 2263 | Detail: "delete of instance " + host + " accepted, operation " + accepted.operation + " (cli)", |
| 2264 | }) |
| 2265 | s.bumpSerial(ctx) |
| 2266 | } |
| 2267 | if s.writeClaimPending(w, err) { |
| 2268 | return |
| 2269 | } |
| 2270 | log.Printf("console: api instances delete %s/%s/%s: %v", acct.Sqid, app, host, err) |
| 2271 | jsonError(w, http.StatusBadGateway, "could not delete the instance — try again") |
| 2272 | return |
| 2273 | } |
| 2274 | // DNS seam (spec §10): a multi-region deployment retires the instance's |
| 2275 | // CNAME here, after the record is gone. |
| 2276 | // |
| 2277 | // Audited whether or not an object was actually there to remove: |
| 2278 | // idempotency makes "deleted" and "already gone" indistinguishable from |
| 2279 | // the store, and the fact worth recording is that this principal asked |
| 2280 | // for this host to stop existing. |
| 2281 | s.auditActivity(acct, app, principal, Activity{ |
| 2282 | Action: "instance-delete", TargetType: "instance", Target: host, Message: message, |
| 2283 | Detail: "deleted instance " + host + " (cli)", |
| 2284 | }) |
| 2285 | // Retiring a route is reconcile's work too, and the direction people want |
| 2286 | // FAST: the announcement is what takes the route down in seconds. |
| 2287 | s.bumpSerial(ctx) |
| 2288 | writeJSON(w, http.StatusOK, map[string]any{"result": "deleted"}) |
| 2289 | } |
| 2290 | |
| 2291 | // handleAPIInstanceToken mints the app-held credential (spec §8). Gated by |
| 2292 | // apiPerson and then apiAccountApp DIRECTLY, not by the instance gate: |
| 2293 | // minting is a human act, and a credential may never mint a credential — |
| 2294 | // including itself. A service principal meets the 401 a fabricated token |
| 2295 | // meets, the same one it met when this door's only resolver was cliUser. |
| 2296 | // Budgeted on the address-keyed write limiter for the same reason: the |
| 2297 | // principal a mint belongs to is a person. |
| 2298 | // |
| 2299 | // That 401 is the INNER of two walls, and it is the one nothing can reach: |
| 2300 | // the outer is cliRoute, where this door's zero Set refuses every role, |
| 2301 | // admin included (TestNoRoleReachesAMintFleetOrPolicyDoor, |
| 2302 | // TestAdminIsRefusedAtAClosedDoor). Both are kept, because the inner one |
| 2303 | // used to be a side effect of resolving through cliUser and would otherwise |
| 2304 | // have vanished the day that resolution became principal-aware. |
| 2305 | func (s *Server) handleAPIInstanceToken(w http.ResponseWriter, r *http.Request) { |
| 2306 | if !s.apiPerson(w, r) { |
| 2307 | return |
| 2308 | } |
| 2309 | addr, acct, app, ok := s.apiAccountApp(w, r) |
| 2310 | if !ok { |
| 2311 | return |
| 2312 | } |
| 2313 | p, _ := principalFrom(r.Context()) |
| 2314 | if !s.allowWrite(p, time.Now()) { |
| 2315 | jsonError(w, http.StatusTooManyRequests, "too many requests — wait a minute and try again") |
| 2316 | return |
| 2317 | } |
| 2318 | var req struct { |
| 2319 | Label string `json:"label"` |
| 2320 | } |
| 2321 | if !readJSON(w, r, &req) { |
| 2322 | return |
| 2323 | } |
| 2324 | // Same de-fanging every terminal label gets: this string is rendered on |
| 2325 | // the minter's tokens page, where it is the one field a human reads to |
| 2326 | // decide whether to revoke. |
| 2327 | label := cleanLabel(req.Label) |
| 2328 | if label == "" { |
| 2329 | label = app + " instance token" |
| 2330 | } |
| 2331 | raw, err := s.store.NewAppTokenContext(r.Context(), acct.ID, app, addr, label, time.Now()) |
| 2332 | if err != nil { |
| 2333 | log.Printf("console: api instance token %s/%s: %v", acct.Sqid, app, err) |
| 2334 | jsonError(w, http.StatusInternalServerError, "could not mint the token — try again") |
| 2335 | return |
| 2336 | } |
| 2337 | s.audit(acct, app, addr, "instance-token", "minted an instance token for "+app+" (cli)") |
| 2338 | // Shown exactly once: only the hash is stored, so there is no second |
| 2339 | // copy to hand out and no way to recover this one. Delivery to the app |
| 2340 | // is the config-env layer, like any other app secret — |
| 2341 | // `carlos secrets set -app <app> CARLOS_INSTANCE_TOKEN=...`. |
| 2342 | writeJSON(w, http.StatusCreated, map[string]any{ |
| 2343 | "token": raw, "account": acct.Sqid, "app": app, |
| 2344 | }) |
| 2345 | } |
| 2346 | |
| 2347 | // instanceSeed bounds (issue #66). Seed becomes the argv the activator execs |
| 2348 | // the app's OWN binary with on first wake — the caller already controls that |
| 2349 | // binary via ship, so the bound here is manifest hygiene, not privilege: a |
| 2350 | // manifest is read forever, and an unbounded argv written once would be |
| 2351 | // permanent. |
| 2352 | const ( |
| 2353 | instanceSeedMaxArgs = 16 |
| 2354 | instanceSeedMaxArgLen = 256 |
| 2355 | ) |
| 2356 | |
| 2357 | // handleAPIInstanceEnable is the member-facing half of `carlos instances |
| 2358 | // enable` (issue #66): the one bucket-only prerequisite that kept the |
| 2359 | // otherwise-API path from "promoted" to "reachable" needing a box shell. |
| 2360 | // |
| 2361 | // The security line it holds: the manifest's DOMAIN is pinned server-side to |
| 2362 | // <account-sqid>.<AppsDomain> — the account's own platform subdomain — and |
| 2363 | // nothing the request carries can name another. Custom domains (tito's |
| 2364 | // go.tito.io) stay an operator act in store mode, and an existing manifest |
| 2365 | // under any other domain is refused with a 409, never overwritten: this door |
| 2366 | // can hand an account keys only to hostnames that were already hers. |
| 2367 | // |
| 2368 | // Gated by apiPerson and then apiAccountApp DIRECTLY, not the instance gate: |
| 2369 | // enabling is policy, a human act — a service credential meets the 401 here |
| 2370 | // that it meets at the token mint, and for the same reason. The manifest |
| 2371 | // this door writes is standing policy about what may ever be BROUGHT INTO |
| 2372 | // existence for the app; a credential the app itself holds must not be able |
| 2373 | // to rewrite the bounds it runs inside. Budgeted on the address-keyed write |
| 2374 | // limiter for the same reason. |
| 2375 | // |
| 2376 | // The door is registered with the zero Set (authcli.go), so no role reaches |
| 2377 | // it and a service credential is refused out at cliRoute with the same 403 |
| 2378 | // the mint door gives. That registration is the load-bearing wall; apiPerson |
| 2379 | // is the second one, stated here because the person-only rule is a fact |
| 2380 | // about the SUBJECT of this door rather than about any caller's role, and |
| 2381 | // because it used to hold only as a side effect of resolving through |
| 2382 | // cliUser. Pinned by TestAPIInstanceEnableRefusals, TestEnableStaysAHumanAct |
| 2383 | // and TestApiPersonIsTheInnerWall. |
| 2384 | func (s *Server) handleAPIInstanceEnable(w http.ResponseWriter, r *http.Request) { |
| 2385 | if !s.apiPerson(w, r) { |
| 2386 | return |
| 2387 | } |
| 2388 | addr, acct, app, ok := s.apiAccountApp(w, r) |
| 2389 | if !ok { |
| 2390 | return |
| 2391 | } |
| 2392 | p, _ := principalFrom(r.Context()) |
| 2393 | if !s.allowWrite(p, time.Now()) { |
| 2394 | jsonError(w, http.StatusTooManyRequests, "too many requests — wait a minute and try again") |
| 2395 | return |
| 2396 | } |
| 2397 | if s.cfg.Releases == nil { |
| 2398 | jsonError(w, http.StatusServiceUnavailable, |
| 2399 | "no deployment bucket on this host — the console is running without one") |
| 2400 | return |
| 2401 | } |
| 2402 | if s.cfg.AppsDomain == "" { |
| 2403 | // Membership is proven, so this may name the operator act that fixes |
| 2404 | // it. 503, not 409: the missing piece is deployment configuration |
| 2405 | // (PLATFORM_APPS_DOMAIN on the console), not anything about this app. |
| 2406 | jsonError(w, http.StatusServiceUnavailable, |
| 2407 | "this console has no platform apps domain configured — an operator enables instances in store mode (carlos instances enable) or sets PLATFORM_APPS_DOMAIN on the console") |
| 2408 | return |
| 2409 | } |
| 2410 | // Channel and Seed are pointers because "absent" and "empty" are |
| 2411 | // different requests on the wire (the apiConfigApply precedent): an |
| 2412 | // omitted field preserves what the manifest already carries, an explicit |
| 2413 | // value replaces it — including seed's explicit [] meaning "clear". |
| 2414 | // |
| 2415 | // PrivateStorage joined them on 2026-09-22 (issue #412) as a pointer for |
| 2416 | // exactly the same reason: an app that already gives its instances their |
| 2417 | // own storage must not lose the policy to a `carlos instances enable |
| 2418 | // -channel beta` that never mentioned it. |
| 2419 | var req struct { |
| 2420 | Channel *string `json:"channel"` |
| 2421 | Seed *[]string `json:"seed"` |
| 2422 | PrivateStorage *bool `json:"private_storage"` |
| 2423 | } |
| 2424 | if !readJSON(w, r, &req) { |
| 2425 | return |
| 2426 | } |
| 2427 | if req.Channel != nil && !release.ValidChannel(*req.Channel) { |
| 2428 | jsonError(w, http.StatusBadRequest, "invalid channel name") |
| 2429 | return |
| 2430 | } |
| 2431 | if req.Seed != nil { |
| 2432 | if len(*req.Seed) > instanceSeedMaxArgs { |
| 2433 | jsonError(w, http.StatusBadRequest, "seed takes at most 16 arguments") |
| 2434 | return |
| 2435 | } |
| 2436 | for _, arg := range *req.Seed { |
| 2437 | if arg == "" || len(arg) > instanceSeedMaxArgLen { |
| 2438 | jsonError(w, http.StatusBadRequest, "each seed argument must be 1-256 bytes") |
| 2439 | return |
| 2440 | } |
| 2441 | } |
| 2442 | } |
| 2443 | domain := acct.Sqid + "." + s.cfg.AppsDomain |
| 2444 | // The domain is about to be stamped into a manifest every future host |
| 2445 | // gets checked against, and ValidHost only regex-checks the leading |
| 2446 | // label — the suffix rides into bucket keys verbatim. Store mode |
| 2447 | // validates its -domain for exactly this reason (instances_cli.go); a |
| 2448 | // misconfigured PLATFORM_APPS_DOMAIN ("*.oncarlos.com" copied off a DNS |
| 2449 | // record) must refuse here, not surface as records reconcile skips |
| 2450 | // forever. The console also refuses garbage at boot (cmd/console); this |
| 2451 | // guard covers embedders that build Config directly. |
| 2452 | if !provision.ValidHostShape(domain) { |
| 2453 | jsonError(w, http.StatusServiceUnavailable, |
| 2454 | "this console's PLATFORM_APPS_DOMAIN is not a valid domain — fix the console configuration") |
| 2455 | return |
| 2456 | } |
| 2457 | ctx, cancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| 2458 | defer cancel() |
| 2459 | // Read WITH the etag and write guarded by it below: the domain-conflict |
| 2460 | // refusal is only honest if nothing can land between this read and the |
| 2461 | // write. PutManifest's upsert re-reads inside, so an operator's |
| 2462 | // store-mode manifest written into that window would be silently |
| 2463 | // re-homed — the exact overwrite the 409 exists to prevent. |
| 2464 | m, etag, err := provision.GetManifestETag(ctx, s.cfg.Releases, acct.Sqid, app) |
| 2465 | switch { |
| 2466 | case errors.Is(err, os.ErrNotExist): |
| 2467 | m, etag = &provision.Manifest{Schema: 1, App: app, Domain: domain}, "" |
| 2468 | case err != nil: |
| 2469 | log.Printf("console: api instances enable read %s/%s: %v", acct.Sqid, app, err) |
| 2470 | jsonError(w, http.StatusBadGateway, "could not read instance settings") |
| 2471 | return |
| 2472 | case m.Domain != domain: |
| 2473 | // Never overwritten from here: a manifest naming any other domain |
| 2474 | // was written by an operator in store mode, and re-homing every |
| 2475 | // FUTURE host check under the platform domain would be this door |
| 2476 | // rewriting operator policy. Membership is proven, so naming the |
| 2477 | // standing domain leaks nothing the caller cannot already read. |
| 2478 | jsonError(w, http.StatusConflict, |
| 2479 | "instances for "+app+" are managed under "+m.Domain+ |
| 2480 | " — changing an app's instance domain is an operator act (carlos instances enable, store mode)") |
| 2481 | return |
| 2482 | } |
| 2483 | // Pool and Regions survive untouched on an update: they are operator |
| 2484 | // policy this door never speaks for, in either direction. (Recorded |
| 2485 | // debt: fields ADDED to schema 1 after this binary would be dropped by |
| 2486 | // this read-modify-write, the same property the store-mode upsert has — |
| 2487 | // an additive manifest field needs the console upgraded before members |
| 2488 | // enable through it.) |
| 2489 | if req.Channel != nil { |
| 2490 | m.Channel = *req.Channel |
| 2491 | } |
| 2492 | if req.Seed != nil { |
| 2493 | m.Seed = *req.Seed |
| 2494 | } |
| 2495 | // Turning the policy ON is checked against what this deployment and this |
| 2496 | // account can actually do; turning it OFF never is (spec §5). The |
| 2497 | // asymmetry is deliberate: a policy that cannot be honoured fails once |
| 2498 | // per signup in a door whose caller is a machine, so the person |
| 2499 | // declaring it is told now — but an account whose entitlement was later |
| 2500 | // declined must be able to get its create door working again without |
| 2501 | // waiting on an operator. |
| 2502 | if req.PrivateStorage != nil && *req.PrivateStorage && |
| 2503 | s.refuseTemporary(w, acct, app, "give its instances private storage") { |
| 2504 | return |
| 2505 | } |
| 2506 | if req.PrivateStorage != nil && *req.PrivateStorage && !m.PrivateStorage { |
| 2507 | if !s.instancePrivateStorageAllowed(ctx, w, acct) { |
| 2508 | return |
| 2509 | } |
| 2510 | } |
| 2511 | if req.PrivateStorage != nil { |
| 2512 | m.PrivateStorage = *req.PrivateStorage |
| 2513 | } |
| 2514 | if err := provision.PutManifestIfMatch(ctx, s.cfg.Releases, acct.Sqid, app, *m, etag); err != nil { |
| 2515 | if errors.Is(err, release.ErrCASConflict) { |
| 2516 | jsonError(w, http.StatusConflict, |
| 2517 | "instance settings changed while this request was in flight — try again") |
| 2518 | return |
| 2519 | } |
| 2520 | log.Printf("console: api instances enable %s/%s: %v", acct.Sqid, app, err) |
| 2521 | jsonError(w, http.StatusBadGateway, "could not enable instances — try again") |
| 2522 | return |
| 2523 | } |
| 2524 | s.audit(acct, app, addr, "instance-enable", "enabled instances under "+domain+privateStorageAuditSuffix(req.PrivateStorage)+" (cli)") |
| 2525 | // The manifest names the channel every minted route follows, so a change |
| 2526 | // here changes what boxes converge onto — reconcile reads it each pass. |
| 2527 | s.bumpSerial(ctx) |
| 2528 | writeJSON(w, http.StatusOK, map[string]any{ |
| 2529 | "app": app, "domain": m.Domain, "pool": m.Pool, "channel": m.Channel, |
| 2530 | "regions": m.Regions, "seed": m.Seed, "private_storage": m.PrivateStorage, |
| 2531 | }) |
| 2532 | } |
| 2533 | |
| 2534 | // privateStorageAuditSuffix names the policy in the audit line only when |
| 2535 | // this request SPOKE to it. An enable that changed a channel says nothing |
| 2536 | // about storage, and a row that restated the standing policy every time |
| 2537 | // would make the one request that changed it impossible to find. |
| 2538 | func privateStorageAuditSuffix(req *bool) string { |
| 2539 | switch { |
| 2540 | case req == nil: |
| 2541 | return "" |
| 2542 | case *req: |
| 2543 | return ", every instance with storage of its own" |
| 2544 | default: |
| 2545 | return ", instances sharing the app's storage" |
| 2546 | } |
| 2547 | } |
| 2548 | |
| 2549 | // instanceRegionAllowed applies the manifest's region allow-list. An empty |
| 2550 | // list means any shape-valid region (spec §3.1) — including the empty, |
| 2551 | // unconstrained one, which is what a single-region deployment uses. |
| 2552 | func instanceRegionAllowed(allowed []string, region string) bool { |
| 2553 | return len(allowed) == 0 || slices.Contains(allowed, region) |
| 2554 | } |
| 2555 | |
amadan