amadan
#162: console doors for replica creation, the opt-in and its refusals
The create door mints a replica's host, stamps its home's channel on it, and refuses every cross-record shape spec §6 names: a home that does not exist, is itself a replica, is not latency-steered, or never opted into the placement asked for; a caller-supplied host or channel; the home's own placement; the deployment's home region; a fleet-placed app. The PATCH door owns the opt-in — the manifest allow-list, the armed-and- hosting gate, and the refusal to withdraw a placement that still holds a replica record or to turn steering off underneath one. Delete refuses a home with replicas and never fails open on a store error. Move refuses a replica in moveGate, so every move verb inherits it, and refuses a home moving into one of its own replica placements. Role and Home join protectedFieldsEqual; Replicas does not, because the PATCH door legitimately changes it and the guard would refuse the door it exists to protect.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
5 files changed,
+669
−4
internal/console/apiinstancemove.go+24 −1internal/console/apiinstances.go+314 −3internal/console/apiinstances_test.go+277 −0internal/provision/maintenance.go+15 −0internal/provision/maintenance_test.go+39 −0
CI passed — run details
diff --git a/internal/console/apiinstancemove.go b/internal/console/apiinstancemove.go| index 0c7466c91..a0299f7c9 100644 |
| --- a/internal/console/apiinstancemove.go |
| +++ b/internal/console/apiinstancemove.go |
| @@ -50,6 +50,17 @@ func (s *Server) moveGate(w http.ResponseWriter, r *http.Request) (acct *Account |
| jsonError(w, http.StatusBadGateway, "could not read the instance") |
| return nil, "", "", nil, false |
| } |
| + // A replica is a mirror OF something, and what it mirrors is fixed at |
| + // create (spec §2): repointing it is delete-and-recreate, because the |
| + // home's own placement, its channel and the alias the replica publishes |
| + // are all derived from the pair. The rail lives here, in the gate every |
| + // move verb shares, so cut and abort inherit it rather than each |
| + // carrying a copy that could drift. |
| + if rec.Role == "replica" { |
| + jsonError(w, http.StatusBadRequest, |
| + "a replica does not move — it mirrors "+rec.Home+" from the placement it was created in; delete it, once its data has drained, and create a replica in the placement you want") |
| + return nil, "", "", nil, false |
| + } |
| if k := rec.Kind; k != "" && k != "instance" { |
| jsonError(w, http.StatusBadRequest, "only exec-backed instances move — this route is kind "+k) |
| return nil, "", "", nil, false |
| @@ -177,7 +188,7 @@ func (s *Server) handleAPIInstanceMoveStatus(w http.ResponseWriter, r *http.Requ |
| } |
| func (s *Server) handleAPIInstanceMoveRequest(w http.ResponseWriter, r *http.Request) { |
| - acct, app, host, _, ok := s.moveGate(w, r) |
| + acct, app, host, rec, ok := s.moveGate(w, r) |
| if !ok { |
| return |
| } |
| @@ -200,6 +211,18 @@ func (s *Server) handleAPIInstanceMoveRequest(w http.ResponseWriter, r *http.Req |
| jsonError(w, http.StatusBadRequest, "moves to the home region are not supported yet — the home publishes no staging lane") |
| return |
| } |
| + // A home moving INTO one of its own replica placements would land its |
| + // route and its replica's alias for the account's public host on one |
| + // box, which is a collision neither pass can resolve (spec §6). The |
| + // comparison goes through homeRegionOut on both sides, like every other |
| + // placement comparison this feature makes. |
| + for _, code := range rec.Replicas { |
| + if s.homeRegionOut(code) == s.homeRegionOut(to) { |
| + jsonError(w, http.StatusBadRequest, |
| + "region "+s.homeRegionOut(to)+" is one of this instance's own replica placements — its route and the replica's alias for the same host would collide on that box; withdraw the placement first") |
| + return |
| + } |
| + } |
| // The armed gate is MANDATORY for moves — unlike create there is no |
| // dormant path, because a move needs a live target box to restore on; |
| // a deployment with no armed placements cannot be a move target at all. |
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go| index 1256c0024..f7a0beb8f 100644 |
| --- a/internal/console/apiinstances.go |
| +++ b/internal/console/apiinstances.go |
| @@ -28,6 +28,7 @@ package console |
| import ( |
| "context" |
| + "crypto/rand" |
| "encoding/json" |
| "errors" |
| "log" |
| @@ -615,6 +616,13 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| Environment string `json:"environment"` |
| Addrs []string `json:"addrs"` |
| Health *provision.Health `json:"probe"` |
| + // Role and Home are the replica pair (#162, spec §6). Replicas is |
| + // NOT here: the opt-in is a PATCH on a home that already exists and |
| + // is already steered, never a create-time field — a record cannot |
| + // be born carrying placements its own steering has not been proved |
| + // against. |
| + Role string `json:"role"` |
| + Home string `json:"home"` |
| } |
| if !readJSON(w, r, &req) { |
| return |
| @@ -623,7 +631,45 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| if !ok { |
| return |
| } |
| + // The role is settled FIRST, ahead of the host check below, because a |
| + // replica has no host to check: the platform mints it (spec §6) and the |
| + // caller who supplied one is refused here rather than being told its |
| + // shape is wrong. Everything a replica needs from ANOTHER record — the |
| + // home, its steering, its opt-in — waits until `placed` and `region` |
| + // exist, further down. |
| + role := strings.TrimSpace(req.Role) |
| + if !provision.ValidRole(role) { |
| + jsonError(w, http.StatusBadRequest, "role "+role+": want \"\" or \"replica\"") |
| + return |
| + } |
| + if role == "replica" { |
| + if strings.TrimSpace(req.Host) != "" { |
| + jsonError(w, http.StatusBadRequest, |
| + "a replica's host is minted by the platform — drop host") |
| + return |
| + } |
| + if strings.TrimSpace(req.Channel) != "" { |
| + jsonError(w, http.StatusBadRequest, |
| + "a replica runs its home's channel — drop channel") |
| + return |
| + } |
| + } else if strings.TrimSpace(req.Home) != "" { |
| + // `home` is the replica half of the pair and means nothing without |
| + // it. Refused rather than dropped: a caller who sent one believes |
| + // this record mirrors something, and a silently ignored field would |
| + // leave them with a record that does not. |
| + jsonError(w, http.StatusBadRequest, |
| + "home names the instance a REPLICA mirrors — send role replica too, or drop home") |
| + return |
| + } |
| host := strings.TrimSpace(req.Host) |
| + if role == "replica" { |
| + // A name of its own, not a secret: it is published in the box's |
| + // edges file by design (spec §6). Minted before the shape check |
| + // below so a replica passes the SAME one-label rail every other |
| + // record does, with no branch of its own. |
| + host = mintReplicaHost(m.Domain) |
| + } |
| // Path-traversal defence stated as such (spec §11): this host becomes a |
| // bucket key segment here and, on the box, the <host>.sock and |
| // <host>.db path segments. Nothing below runs until it is one label |
| @@ -754,6 +800,65 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| return |
| } |
| } |
| + // The replica's CROSS-RECORD rails, which no other door in this file |
| + // has and nothing downstream repeats: ValidateShape sees one record and |
| + // cannot know whether the home exists, is steered, or opted into this |
| + // placement, and the reconcile pass refuses none of it (spec §6). They |
| + // wait until here because they are about `placed` and `region`, both of |
| + // which are settled above. |
| + var homeRec *provision.Record |
| + if role == "replica" { |
| + if placed != "" { |
| + jsonError(w, http.StatusBadRequest, |
| + "this app is placed on a fleet, so every instance inherits the fleet's placement and a replica has none of its own to choose — unplace the app or drop role") |
| + return |
| + } |
| + // "" is the deployment's HOME region on a record (homeRegionIn), and |
| + // a replica record with an empty region is refused by ValidateShape. |
| + // Said here in words a caller can act on rather than as "no region": |
| + // a replica cannot live in the deployment's home region at all |
| + // (spec §2). |
| + if region == "" { |
| + jsonError(w, http.StatusBadRequest, |
| + "region "+s.homeRegionOut(region)+" is this deployment's home region, and a replica lives in a named placement somewhere else — a replica cannot be created there") |
| + return |
| + } |
| + homeHost := strings.TrimSpace(req.Home) |
| + rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, homeHost) |
| + switch { |
| + case errors.Is(err, os.ErrNotExist): |
| + jsonError(w, http.StatusBadRequest, "home "+homeHost+": no such instance in this app") |
| + return |
| + case err != nil: |
| + log.Printf("console: api instances create %s/%s: read home %s: %v", acct.Sqid, app, homeHost, err) |
| + jsonError(w, http.StatusBadGateway, "could not read this instance's home — try again") |
| + return |
| + } |
| + if rec.Role != "" { |
| + jsonError(w, http.StatusBadRequest, |
| + "home "+homeHost+" is itself a replica — a replica of a replica is not a shape the platform carries") |
| + return |
| + } |
| + if rec.Steering != "latency" { |
| + jsonError(w, http.StatusBadRequest, |
| + "home "+homeHost+" is not latency-steered, so its host resolves to one edge and a replica would serve nothing — set steering first") |
| + return |
| + } |
| + // Both sides go through homeRegionOut: a home in the deployment's |
| + // own region carries "" and would otherwise compare equal to |
| + // nothing at all (fable B-5). |
| + if s.homeRegionOut(region) == s.homeRegionOut(rec.Region) { |
| + jsonError(w, http.StatusBadRequest, |
| + "region "+s.homeRegionOut(region)+" is the home's own placement — a replica lives somewhere else") |
| + return |
| + } |
| + if !slices.Contains(rec.Replicas, region) { |
| + jsonError(w, http.StatusBadRequest, |
| + "region "+s.homeRegionOut(region)+" is not one this account opted into — set replicas on "+rec.Host+" first") |
| + return |
| + } |
| + homeRec = rec |
| + } |
| channel := strings.TrimSpace(req.Channel) |
| if channel == "" { |
| resolved, ref := s.instanceChannel(ctx, acct.Sqid, app, m) |
| @@ -763,6 +868,14 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| } |
| channel = resolved |
| } |
| + // A replica runs its home's build, and the record cannot say "whatever |
| + // the home runs" by leaving the field empty: Reconcile protects a |
| + // channel-less record before scope is even asked, so it would mint |
| + // nowhere (spec §2). The home's value is stamped on, after the |
| + // caller-supplied one was refused above. |
| + if homeRec != nil { |
| + channel = homeRec.Channel |
| + } |
| if !release.ValidChannel(channel) { |
| jsonError(w, http.StatusBadRequest, "invalid channel name") |
| return |
| @@ -788,6 +901,14 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| Environment: strings.TrimSpace(req.Environment), |
| Addrs: req.Addrs, |
| Health: req.Health, |
| + Role: role, |
| + } |
| + if homeRec != nil { |
| + // The home's OWN spelling of its host, read back from the record |
| + // the rails above were checked against — not the request's, which |
| + // is only ever a lookup key. routes.replica_of is joined on this |
| + // value on the box. |
| + rec.Home = homeRec.Host |
| } |
| // A static platform host is HA by default. Static is the one serving |
| // class every platform box materializes unconditionally |
| @@ -837,8 +958,28 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| jsonError(w, http.StatusBadRequest, err.Error()) |
| return |
| } |
| - if err := provision.CreateRecord(ctx, s.cfg.Releases, rec); err != nil { |
| + // CreateRecord is create-only and 409s on a race. A caller-chosen host |
| + // has nothing to retry — the name was theirs and the conflict IS the |
| + // answer — but a minted one does: try a fresh name twice, then give up |
| + // (spec §6). The name is baked into three places by now (Host, Slug and |
| + // the shape check), so all three are re-done rather than the string |
| + // alone, and the response carries whichever name won. |
| + for attempt := 0; ; attempt++ { |
| + err := provision.CreateRecord(ctx, s.cfg.Releases, rec) |
| + if err == nil { |
| + break |
| + } |
| if errors.Is(err, release.ErrCASConflict) { |
| + if role == "replica" && attempt < 2 { |
| + rec.Host = mintReplicaHost(m.Domain) |
| + rec.Slug = strings.TrimSuffix(rec.Host, "."+m.Domain) |
| + host = rec.Host |
| + if verr := provision.ValidateShape(rec); verr != nil { |
| + jsonError(w, http.StatusBadRequest, verr.Error()) |
| + return |
| + } |
| + continue |
| + } |
| // A lost race against a second signup for the same host, or a |
| // plain duplicate. Either way the winner's record stands. |
| jsonError(w, http.StatusConflict, "instance "+host+" already exists") |
| @@ -1101,13 +1242,18 @@ func (s *Server) handleAPIInstanceUpdate(w http.ResponseWriter, r *http.Request) |
| // here, never stored or echoed (spec §5.2). The plaintext crosses |
| // the wire once, over TLS, the same as a sign-in. |
| Gate *string `json:"gate"` |
| + // Replicas is a *[]string for Addrs' reason: absent leaves the |
| + // opt-in alone, `"replicas":[]` withdraws every placement, and the |
| + // two are different requests. This is the ONLY door that writes the |
| + // field — a create cannot carry it (spec §6). |
| + Replicas *[]string `json:"replicas"` |
| } |
| if !readJSON(w, r, &req) { |
| return |
| } |
| - if req.Addrs == nil && req.Probe == nil && req.Steering == nil && req.Environment == nil && req.Gate == nil { |
| + if req.Addrs == nil && req.Probe == nil && req.Steering == nil && req.Environment == nil && req.Gate == nil && req.Replicas == nil { |
| jsonError(w, http.StatusBadRequest, |
| - "nothing to update — send addrs, probe, steering, environment, gate, or a combination") |
| + "nothing to update — send addrs, probe, steering, environment, gate, replicas, or a combination") |
| return |
| } |
| var probe *provision.Health |
| @@ -1154,6 +1300,79 @@ func (s *Server) handleAPIInstanceUpdate(w http.ResponseWriter, r *http.Request) |
| if req.Environment != nil { |
| rec.Environment = strings.TrimSpace(*req.Environment) |
| } |
| + // The residence opt-in (#162, spec §6). Every rail here is a |
| + // cross-record one — what this app's manifest allows, what this |
| + // deployment's boxes can host, and which placements already hold a |
| + // replica RECORD — so none of it can be delegated to ValidateShape, |
| + // which sees this record alone. |
| + if req.Replicas != nil { |
| + if rec.Role == "replica" { |
| + jsonError(w, http.StatusBadRequest, |
| + "a replica does not carry replicas of its own — the opt-in lives on its home, "+rec.Home) |
| + return |
| + } |
| + // The manifest is not in scope in this handler and is not read on |
| + // any other path through it: read it here, where it is needed. |
| + // instanceManifest writes its own refusal. |
| + m, mok := s.instanceManifest(ctx, w, acct, app) |
| + if !mok { |
| + return |
| + } |
| + norm, err := provision.ValidReplicas(*req.Replicas) |
| + if err != nil { |
| + jsonError(w, http.StatusBadRequest, err.Error()) |
| + return |
| + } |
| + caps, published := placements(ctx, s.cfg.Releases) |
| + for _, code := range norm { |
| + // Both sides through homeRegionOut, for the reason the create |
| + // door gives: a home in the deployment's own region carries "" |
| + // and a literal comparison would never fire (fable B-5). |
| + if s.homeRegionOut(code) == s.homeRegionOut(rec.Region) { |
| + jsonError(w, http.StatusBadRequest, |
| + "replica placement "+code+" is this instance's own placement — a replica lives somewhere else") |
| + return |
| + } |
| + if !instanceRegionAllowed(m.Regions, code) { |
| + jsonError(w, http.StatusBadRequest, |
| + "replica placement "+code+" is not one this app runs in — allowed: "+strings.Join(m.Regions, ", ")) |
| + return |
| + } |
| + // Dormant on a fleet that publishes no placement codes, the |
| + // same bar and the same silence the create door keeps. |
| + if published && !caps[code].Hosting { |
| + jsonError(w, http.StatusBadRequest, |
| + "replica placement "+code+" cannot host an instance in this deployment — regions that can: "+armedCodes(hostingSet(caps))) |
| + return |
| + } |
| + } |
| + // Withdrawing a placement that still holds a replica record would |
| + // leave the account's order data living somewhere its owner has |
| + // stopped agreeing to. The replica is deleted first; the platform |
| + // cannot see its drain state and does not guess. |
| + gone, err := replicaPlacementsInUse(ctx, s.cfg.Releases, acct.Sqid, app, rec.Host, norm) |
| + if err != nil { |
| + log.Printf("console: api instances update %s/%s/%s: list replicas: %v", acct.Sqid, app, host, err) |
| + jsonError(w, http.StatusBadGateway, "could not check this instance's replicas — try again") |
| + return |
| + } |
| + if len(gone) > 0 { |
| + jsonError(w, http.StatusConflict, |
| + "a replica still lives in "+strings.Join(gone, ", ")+" — delete the replica instance first, once its data has drained") |
| + return |
| + } |
| + rec.Replicas = norm |
| + } |
| + // The same invariant from the other side: the opt-in depends on latency |
| + // steering (spec §5), so steering cannot be turned off underneath a live |
| + // set. ValidateShape refuses the pair on the way in; this refuses the |
| + // half that would otherwise dismantle it. |
| + if req.Steering != nil && *req.Steering != "latency" && len(rec.Replicas) > 0 { |
| + jsonError(w, http.StatusConflict, |
| + "this instance has replica placements in "+strings.Join(rec.Replicas, ", ")+ |
| + ", which depend on latency steering — withdraw the placements first") |
| + return |
| + } |
| // The gate is the one field this door does not store as it was sent: the |
| // record carries a bcrypt hash and nothing else, so the plaintext lives |
| // only in this stack frame. Not trimmed either — a password's leading and |
| @@ -1264,6 +1483,11 @@ func (s *Server) handleAPIInstanceUpdate(w http.ResponseWriter, r *http.Request) |
| if req.Environment != nil { |
| changed = append(changed, "environment") |
| } |
| + // "replicas" is exactly the kind of fact this log exists for: it names |
| + // where an account's order data is allowed to live. |
| + if req.Replicas != nil { |
| + changed = append(changed, "replicas") |
| + } |
| // "gate" names that the host's password was set or cleared, never which |
| // it was: the audit log answers "did anyone change the gate on this host |
| // last week?" without becoming a place a password can be read. |
| @@ -1327,6 +1551,43 @@ func (s *Server) handleAPIInstanceDelete(w http.ResponseWriter, r *http.Request) |
| jsonError(w, http.StatusBadRequest, "invalid host") |
| return |
| } |
| + // A HOME with replicas is refused (spec §6): its mirrors would go on |
| + // serving the account's public host from a route whose record has gone, |
| + // and the account's data would be living in placements nothing links |
| + // back to an owner. Deleting the REPLICA is the caller's to sequence |
| + // once its data has drained — the platform cannot see drain state. |
| + // |
| + // This read must not fail OPEN. A store hiccup that skipped the check |
| + // would let exactly the delete this guards against through, so anything |
| + // but "no such record" is a 502; a record that is already gone is the |
| + // idempotent case and has nothing to refuse. |
| + rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host) |
| + switch { |
| + case errors.Is(err, os.ErrNotExist): |
| + // Already gone: the caller's goal is "not there". |
| + case err != nil: |
| + log.Printf("console: api instances delete read %s/%s/%s: %v", acct.Sqid, app, host, err) |
| + jsonError(w, http.StatusBadGateway, "could not read the instance — try again") |
| + return |
| + case rec.Role == "": |
| + live, err := replicaPlacementsInUse(ctx, s.cfg.Releases, acct.Sqid, app, host, nil) |
| + if err != nil { |
| + log.Printf("console: api instances delete %s/%s/%s: list replicas: %v", acct.Sqid, app, host, err) |
| + jsonError(w, http.StatusBadGateway, "could not check this instance's replicas — try again") |
| + return |
| + } |
| + // The opt-in alone is enough: a placement this account still agrees |
| + // to is a decision to withdraw deliberately, with `replicas`, not |
| + // one to lose as a side effect of deleting the home. |
| + named := append(live, rec.Replicas...) |
| + slices.Sort(named) |
| + if named = slices.Compact(named); len(named) > 0 { |
| + jsonError(w, http.StatusConflict, |
| + "this instance has replica placements in "+strings.Join(named, ", ")+ |
| + " — delete those replicas and withdraw the placements first") |
| + return |
| + } |
| + } |
| if err := provision.DeleteRecord(ctx, s.cfg.Releases, acct.Sqid, app, host); err != nil { |
| log.Printf("console: api instances delete %s/%s/%s: %v", acct.Sqid, app, host, err) |
| jsonError(w, http.StatusBadGateway, "could not delete the instance — try again") |
| @@ -1567,6 +1828,56 @@ func (s *Server) handleAPIInstanceEnable(w http.ResponseWriter, r *http.Request) |
| }) |
| } |
| +// mintReplicaHost returns one label under domain for a replica instance: |
| +// "r" plus 10 characters of crypto/rand base32 (Crockford, lower case), so |
| +// it passes the same one-label rail (ValidHost) and the same slug derivation |
| +// every other instance host does. |
| +// |
| +// NOT a secret and nothing depends on it being unguessable — it is published |
| +// in every box's edges file by design (spec §6). crypto/rand is used because |
| +// the alternative, a counter or a timestamp, would make two consoles racing |
| +// the same second collide predictably; the create is create-only and retries. |
| +func mintReplicaHost(domain string) string { |
| + // Crockford base32 without the letters that read as digits: a host a |
| + // person may have to type into a terminal or read off a screen. |
| + const alphabet = "0123456789abcdefghjkmnpqrstvwxyz" |
| + b := make([]byte, 10) |
| + if _, err := rand.Read(b); err != nil { |
| + // crypto/rand.Read never returns an error on any platform this |
| + // runs on; if it ever did, minting a weaker name would be worse. |
| + panic("replica host: " + err.Error()) |
| + } |
| + for i, v := range b { |
| + b[i] = alphabet[int(v)%len(alphabet)] |
| + } |
| + return "r" + string(b) + "." + domain |
| +} |
| + |
| +// replicaPlacementsInUse returns the placements of `home` that still hold a |
| +// replica record and are NOT in `keep`. |
| +// |
| +// A truncated or failed listing is an ERROR, never an empty answer (fable |
| +// B-13): an empty answer from a partial view would let a residence opt-in be |
| +// withdrawn out from under live order data, which is the one failure the |
| +// Replicas field exists to prevent. The caller turns it into a 502. |
| +func replicaPlacementsInUse(ctx context.Context, store release.Store, acct, app, home string, keep []string) ([]string, error) { |
| + recs, truncated, err := provision.ListRecords(ctx, store, acct, app, instanceListMax) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if truncated { |
| + return nil, errors.New("instance list truncated") |
| + } |
| + var out []string |
| + for _, r := range recs { |
| + if r.Role == "replica" && r.Home == home && !slices.Contains(keep, r.Region) { |
| + out = append(out, r.Region) |
| + } |
| + } |
| + slices.Sort(out) |
| + return slices.Compact(out), nil |
| +} |
| + |
| // instanceRegionAllowed applies the manifest's region allow-list. An empty |
| // list means any shape-valid region (spec §3.1) — including the empty, |
| // unconstrained one, which is what a single-region deployment uses. |
diff --git a/internal/console/apiinstances_test.go b/internal/console/apiinstances_test.go| index 2c02e9063..d2c00ee7a 100644 |
| --- a/internal/console/apiinstances_test.go |
| +++ b/internal/console/apiinstances_test.go |
| @@ -2605,3 +2605,280 @@ func TestAPIInstanceListDBBytes(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// --------------------------------------------------------------------------- |
| +// Replica placements (#162, spec §6): the console doors are the ONLY place |
| +// the cross-record rails can live. ValidateShape sees one record, and no |
| +// reconcile pass refuses anything — so every rule below ("this home is |
| +// steered", "this placement was opted into", "this placement still holds a |
| +// replica") is a refusal at the door or it does not exist. |
| +// |
| +// The fixture manifest is replicaManifest(), not titoManifest(): titogo's |
| +// manifest carries Pool "tito", which the PATCH door refuses `steering: |
| +// latency` on, and Regions ["ie"], which refuses "uk" at the allow-list. |
| +// Either one would refuse these requests BEFORE any replica rail was |
| +// reached, which would make the tests pass while proving nothing. |
| +// "de" is in the allow-list and deliberately NOT opted into, so the |
| +// not-opted-in case tests the rail it is named for. |
| + |
| +func replicaManifest() provision.Manifest { |
| + m := titoManifest() |
| + m.Pool = "" |
| + m.Regions = []string{"ie", "uk", "de"} |
| + return m |
| +} |
| + |
| +func createInstanceRaw(t *testing.T, fx *consoleFixture, tok, body string) (int, string) { |
| + t.Helper() |
| + code, _, out := rawReq(t, http.MethodPost, instancesURL(fx, "bab", "titogo"), tok, body) |
| + return code, out |
| +} |
| + |
| +// createInstance is createInstanceRaw for the calls a test needs to SUCCEED, |
| +// and it answers the host the door minted — which for a replica is the only |
| +// way the caller learns it (spec §6). |
| +func createInstance(t *testing.T, fx *consoleFixture, tok, body string) string { |
| + t.Helper() |
| + code, out := createInstanceRaw(t, fx, tok, body) |
| + if code != http.StatusCreated { |
| + t.Fatalf("create %s = %d %s, want 201", body, code, out) |
| + } |
| + var got struct { |
| + Host string `json:"host"` |
| + } |
| + if err := json.Unmarshal([]byte(out), &got); err != nil { |
| + t.Fatalf("create body %s: %v", out, err) |
| + } |
| + return got.Host |
| +} |
| + |
| +func patchInstanceRaw(t *testing.T, fx *consoleFixture, tok, host, body string) (int, string) { |
| + t.Helper() |
| + code, _, out := rawReq(t, http.MethodPatch, updateURL(fx, "bab", "titogo", host), tok, body) |
| + return code, out |
| +} |
| + |
| +func patchInstance(t *testing.T, fx *consoleFixture, tok, host, body string) { |
| + t.Helper() |
| + if code, out := patchInstanceRaw(t, fx, tok, host, body); code != http.StatusOK { |
| + t.Fatalf("patch %s %s = %d %s, want 200", host, body, code, out) |
| + } |
| +} |
| + |
| +func deleteInstanceRaw(t *testing.T, fx *consoleFixture, tok, host string) (int, string) { |
| + t.Helper() |
| + code, _, out := rawReq(t, http.MethodDelete, updateURL(fx, "bab", "titogo", host), tok, "") |
| + return code, out |
| +} |
| + |
| +// replicaHome creates a home record and opts it into the placements named, |
| +// which takes two PATCHes because replicas without latency steering is a |
| +// shape ValidateShape itself refuses. |
| +func replicaHome(t *testing.T, fx *consoleFixture, tok, host, channel string, replicas ...string) string { |
| + t.Helper() |
| + home := createInstance(t, fx, tok, `{"host":"`+host+`","region":"ie","channel":"`+channel+`"}`) |
| + patchInstance(t, fx, tok, home, `{"steering":"latency"}`) |
| + if len(replicas) > 0 { |
| + body, err := json.Marshal(map[string]any{"replicas": replicas}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + patchInstance(t, fx, tok, home, string(body)) |
| + } |
| + return home |
| +} |
| + |
| +// TestAPIInstanceCreateReplica: the create door mints a replica of a steered, |
| +// opted-in home — a host of its own that the caller never chose, on the |
| +// HOME's channel — and refuses every shape spec §6 names. Each refusal case |
| +// differs from the accepted one in exactly the field its rail is about, so a |
| +// case cannot be passing on the manifest allow-list instead. |
| +func TestAPIInstanceCreateReplica(t *testing.T) { |
| + fx, tok := instanceFixture(t, "code-inst-replica-create") |
| + enableInstances(t, fx, "titogo", replicaManifest()) |
| + // The home's channel is "edge" while the MANIFEST's is "stable", so a |
| + // replica carrying "edge" proves it followed the home and not the |
| + // manifest default (spec §2, the channel rule). |
| + home := replicaHome(t, fx, tok, "acme.go.tito.io", "edge", "uk") |
| + |
| + code, out := createInstanceRaw(t, fx, tok, `{"role":"replica","home":"acme.go.tito.io","region":"uk"}`) |
| + if code != http.StatusCreated { |
| + t.Fatalf("create replica = %d: %s", code, out) |
| + } |
| + var got provision.Record |
| + if err := json.Unmarshal([]byte(out), &got); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got.Role != "replica" || got.Home != home { |
| + t.Fatalf("create replica returned role=%q home=%q, want replica/%s", got.Role, got.Home, home) |
| + } |
| + if got.Host == "" || got.Host == home || !strings.HasSuffix(got.Host, ".go.tito.io") { |
| + t.Fatalf("create replica returned host %q, want a minted host of its own under the app's domain", got.Host) |
| + } |
| + if got.Region != "uk" { |
| + t.Fatalf("replica region = %q, want uk", got.Region) |
| + } |
| + if got.Channel != "edge" { |
| + t.Fatalf("replica channel = %q, want the HOME's edge — a replica runs its home's build", got.Channel) |
| + } |
| + if len(got.Replicas) != 0 { |
| + t.Fatalf("replica carries replicas %v, want none", got.Replicas) |
| + } |
| + stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", got.Host) |
| + if err != nil { |
| + t.Fatalf("minted replica %s not in the bucket: %v", got.Host, err) |
| + } |
| + if stored.Role != "replica" || stored.Home != home || stored.Channel != "edge" || stored.Region != "uk" { |
| + t.Fatalf("stored replica = %+v, want role replica of %s on edge in uk", stored, home) |
| + } |
| + |
| + // A second home, NOT steered, so the "not steered" rail can be reached |
| + // by a request that differs from the accepted one only in that field. |
| + unsteered := createInstance(t, fx, tok, `{"host":"plain.go.tito.io","region":"ie","channel":"edge"}`) |
| + |
| + for _, c := range []struct{ name, payload, want string }{ |
| + {"replica of a replica", `{"role":"replica","home":"` + got.Host + `","region":"uk"}`, "itself a replica"}, |
| + {"home is not steered", `{"role":"replica","home":"` + unsteered + `","region":"uk"}`, "not latency-steered"}, |
| + {"placement not opted in", `{"role":"replica","home":"acme.go.tito.io","region":"de"}`, "not one this account opted into"}, |
| + {"home's own placement", `{"role":"replica","home":"acme.go.tito.io","region":"ie"}`, "the home's own placement"}, |
| + {"caller-supplied host", `{"role":"replica","home":"acme.go.tito.io","region":"uk","host":"mine.go.tito.io"}`, "minted by the platform"}, |
| + {"caller-supplied channel", `{"role":"replica","home":"acme.go.tito.io","region":"uk","channel":"stable"}`, "runs its home's channel"}, |
| + {"no such home", `{"role":"replica","home":"nope.go.tito.io","region":"uk"}`, "no such instance"}, |
| + {"unknown role", `{"role":"mirror","home":"acme.go.tito.io","region":"uk"}`, "role"}, |
| + } { |
| + code, out := createInstanceRaw(t, fx, tok, c.payload) |
| + if code < 400 { |
| + t.Errorf("%s: create returned %d, want a refusal: %s", c.name, code, out) |
| + continue |
| + } |
| + if !strings.Contains(out, c.want) { |
| + t.Errorf("%s: refused with %s, want the rail that names %q", c.name, out, c.want) |
| + } |
| + } |
| +} |
| + |
| +// TestAPIInstanceCreateReplicaRefusesAPlacedApp: a fleet-placed app OVERRIDES |
| +// the caller's region with the fleet's (apiinstances.go's `if placed != ""`), |
| +// which would silently put the replica somewhere nobody asked for. Refused |
| +// until someone decides what a replica of a placed app means (spec §6). The |
| +// home is real, steered and opted-in, so nothing else in the door can be |
| +// what refuses this. |
| +func TestAPIInstanceCreateReplicaRefusesAPlacedApp(t *testing.T) { |
| + fx, tok := instanceFixture(t, "code-inst-replica-placed") |
| + enableInstances(t, fx, "titogo", replicaManifest()) |
| + replicaHome(t, fx, tok, "acme.go.tito.io", "stable", "uk") |
| + |
| + if status, body := postJSON(t, fx.ts.URL+"/api/cli/fleets/bab/new", |
| + map[string]string{"name": "pi", "bucket": "pi-fleet-data"}, tok); status != http.StatusOK { |
| + t.Fatalf("create fleet = %d %v, want 200", status, body) |
| + } |
| + if status, body := postJSON(t, fx.ts.URL+"/api/cli/apps/bab/titogo/place", |
| + map[string]string{"target": "fleet/pi"}, tok); status != http.StatusOK { |
| + t.Fatalf("place = %d %v, want 200", status, body) |
| + } |
| + |
| + code, out := createInstanceRaw(t, fx, tok, `{"role":"replica","home":"acme.go.tito.io","region":"uk"}`) |
| + if code < 400 || !strings.Contains(out, "placed on a fleet") { |
| + t.Fatalf("replica create on a placed app = %d %s, want the fleet-placement refusal", code, out) |
| + } |
| +} |
| + |
| +// TestAPIInstanceReplicasPatchRails: the opt-in is PATCHed onto the home, and |
| +// every way of making the opt-in and the data disagree is refused — turning |
| +// steering off under a live set, naming a placement the app does not run in |
| +// or the home already lives in, withdrawing a placement that still holds a |
| +// replica record, and deleting a home that has replicas at all. |
| +func TestAPIInstanceReplicasPatchRails(t *testing.T) { |
| + fx, tok := instanceFixture(t, "code-inst-replica-patch") |
| + enableInstances(t, fx, "titogo", replicaManifest()) |
| + home := createInstance(t, fx, tok, `{"host":"acme.go.tito.io","region":"ie","channel":"stable"}`) |
| + |
| + // Unsteered: the account's host resolves to one edge, so a replica would |
| + // serve nothing. Refused before it can be written. |
| + code, out := patchInstanceRaw(t, fx, tok, home, `{"replicas":["uk"]}`) |
| + if code < 400 || !strings.Contains(out, "set steering first") { |
| + t.Fatalf("replicas on an unsteered record = %d %s, want the steering refusal", code, out) |
| + } |
| + |
| + patchInstance(t, fx, tok, home, `{"steering":"latency"}`) |
| + // A body carrying ONLY replicas is a request, not a no-op. |
| + patchInstance(t, fx, tok, home, `{"replicas":["uk"]}`) |
| + stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", home) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !slices.Equal(stored.Replicas, []string{"uk"}) { |
| + t.Fatalf("stored replicas = %v, want [uk]", stored.Replicas) |
| + } |
| + |
| + for _, c := range []struct{ name, payload, want string }{ |
| + {"steering off under a live set", `{"steering":""}`, "latency steering"}, |
| + {"the home's own placement", `{"replicas":["ie"]}`, "own placement"}, |
| + {"a placement the app does not run in", `{"replicas":["fr"]}`, "not one this app runs in"}, |
| + } { |
| + if code, out := patchInstanceRaw(t, fx, tok, home, c.payload); code < 400 || !strings.Contains(out, c.want) { |
| + t.Errorf("%s: patch = %d %s, want the rail that names %q", c.name, code, out, c.want) |
| + } |
| + } |
| + |
| + // A home carrying an opt-in cannot be deleted even before any replica |
| + // record exists: its mirrors-to-be are the account's residence decision. |
| + if code, out := deleteInstanceRaw(t, fx, tok, home); code != http.StatusConflict { |
| + t.Fatalf("delete of a home with an opt-in = %d %s, want 409", code, out) |
| + } |
| + |
| + replica := createInstance(t, fx, tok, `{"role":"replica","home":"`+home+`","region":"uk"}`) |
| + |
| + // A replica carries no opt-in of its own. |
| + if code, out := patchInstanceRaw(t, fx, tok, replica, `{"replicas":["de"]}`); code < 400 || !strings.Contains(out, "replica") { |
| + t.Errorf("replicas on a replica = %d %s, want a refusal", code, out) |
| + } |
| + // Widening the set is fine while uk is kept. |
| + patchInstance(t, fx, tok, home, `{"replicas":["uk","de"]}`) |
| + // Withdrawing uk is not: a record still lives there. |
| + if code, out := patchInstanceRaw(t, fx, tok, home, `{"replicas":["de"]}`); code != http.StatusConflict || !strings.Contains(out, "uk") { |
| + t.Errorf("withdrawing a placement that still holds a replica = %d %s, want a 409 naming uk", code, out) |
| + } |
| + if code, out := patchInstanceRaw(t, fx, tok, home, `{"replicas":[]}`); code != http.StatusConflict { |
| + t.Errorf("clearing the opt-in under a live replica = %d %s, want 409", code, out) |
| + } |
| + if code, out := deleteInstanceRaw(t, fx, tok, home); code != http.StatusConflict || !strings.Contains(out, "uk") { |
| + t.Errorf("delete of a home with replicas = %d %s, want a 409 naming uk", code, out) |
| + } |
| + // The replica itself deletes freely — sequencing its drain is the |
| + // caller's, and the platform cannot see drain state. |
| + if code, out := deleteInstanceRaw(t, fx, tok, replica); code != http.StatusOK { |
| + t.Fatalf("delete of the replica = %d %s, want 200", code, out) |
| + } |
| + patchInstance(t, fx, tok, home, `{"replicas":[]}`) |
| + if code, out := deleteInstanceRaw(t, fx, tok, home); code != http.StatusOK { |
| + t.Fatalf("delete of a home whose opt-in is gone = %d %s, want 200", code, out) |
| + } |
| +} |
| + |
| +// TestAPIInstanceMoveRefusesReplicas: repointing a replica is |
| +// delete-and-recreate by §2's own rule, and moving a HOME into one of its own |
| +// replica placements would land its route and the replica's alias on one box. |
| +// The first rail lives in moveGate, so every move verb inherits it. |
| +func TestAPIInstanceMoveRefusesReplicas(t *testing.T) { |
| + fx, tok := instanceFixture(t, "code-inst-replica-move") |
| + enableInstances(t, fx, "titogo", replicaManifest()) |
| + home := replicaHome(t, fx, tok, "acme.go.tito.io", "stable", "uk") |
| + replica := createInstance(t, fx, tok, `{"role":"replica","home":"`+home+`","region":"uk"}`) |
| + |
| + moveURL := func(host string) string { return updateURL(fx, "bab", "titogo", host) + "/move" } |
| + |
| + code, _, out := rawReq(t, http.MethodPost, moveURL(replica), tok, `{"region":"de"}`) |
| + if code < 400 || !strings.Contains(out, "replica") { |
| + t.Errorf("move of a replica = %d %s, want a refusal naming the role", code, out) |
| + } |
| + // Every verb in the family inherits it, because the rail is in moveGate. |
| + if code, _, out := rawReq(t, http.MethodPost, moveURL(replica)+"/cut", tok, ""); code < 400 || !strings.Contains(out, "replica") { |
| + t.Errorf("move cut of a replica = %d %s, want a refusal naming the role", code, out) |
| + } |
| + |
| + code, _, out = rawReq(t, http.MethodPost, moveURL(home), tok, `{"region":"uk"}`) |
| + if code < 400 || !strings.Contains(out, "uk") { |
| + t.Errorf("move of a home into its own replica placement = %d %s, want a refusal naming uk", code, out) |
| + } |
| +} |
diff --git a/internal/provision/maintenance.go b/internal/provision/maintenance.go| index dc8b6aa90..f1d6b217e 100644 |
| --- a/internal/provision/maintenance.go |
| +++ b/internal/provision/maintenance.go |
| @@ -78,10 +78,25 @@ func DecodeRecord(raw []byte) (Record, error) { |
| // Equality preserves the projection, not permission to change a fenced record. |
| // Durable guard and lifecycle checks belong to the maintenance integration. |
| +// |
| +// Role and Home joined it with replica placements (spec §6). Both are |
| +// create-only by §2 and never converge, so plain equality is exactly right, |
| +// and what it catches is a writer in THIS binary that read a record through |
| +// a decoder without the fields and wrote it back: dropping Home clears the |
| +// replica's link, which clears routes.replica_of, which sweeps the alias the |
| +// account's public host is served by on the replica's box — silently, on the |
| +// next pass. Replicas is deliberately NOT here: the value legitimately |
| +// changes on every `replicas` PATCH, so including it would make this guard |
| +// refuse the door that polices it. A console build that PREDATES the fields |
| +// runs its own copy of this function and cannot be reached by it at all; |
| +// that half is a rollout rule, not code. |
| func protectedFieldsEqual(before, after Record) bool { |
| if before.Account != after.Account || before.App != after.App || before.Host != after.Host || !before.CreatedAt.Equal(after.CreatedAt) { |
| return false |
| } |
| + if before.Role != after.Role || before.Home != after.Home { |
| + return false |
| + } |
| if len(before.Maintenance) == 0 || len(after.Maintenance) == 0 { |
| return len(before.Maintenance) == len(after.Maintenance) |
| } |
diff --git a/internal/provision/maintenance_test.go b/internal/provision/maintenance_test.go| index 177fa8ffa..3293ebb04 100644 |
| --- a/internal/provision/maintenance_test.go |
| +++ b/internal/provision/maintenance_test.go |
| @@ -231,3 +231,42 @@ func TestMalformedProjectionReconcileAndLists(t *testing.T) { |
| t.Fatal("all-list semantics changed") |
| } |
| } |
| + |
| +// TestProtectedFieldsRefuseAnOldWriter: Role and Home are create-only (spec |
| +// §2) and never converge, so any writer in THIS binary that hands |
| +// UpdateRecord a record with them dropped is doing a read-modify-write |
| +// against a decoder that has never heard of them — which would clear a |
| +// replica's link to its home, and with it the route's replica_of and the |
| +// alias the account's host is served by, silently. |
| +// |
| +// Replicas is deliberately NOT here. The PATCH door's own rails police it, |
| +// and a record whose opt-in legitimately differs from the stored one is |
| +// exactly what `{"replicas":["uk"]}` sends: adding it would make the guard |
| +// refuse the door it exists to protect (spec §6, revision 3). |
| +func TestProtectedFieldsRefuseAnOldWriter(t *testing.T) { |
| + replica := Record{Schema: 1, Host: "r7.go.tito.io", Channel: "stable", |
| + Role: "replica", Home: "acme.go.tito.io"} |
| + for _, c := range []struct { |
| + name string |
| + mut func(*Record) |
| + }{ |
| + {"role dropped", func(r *Record) { r.Role = "" }}, |
| + {"home dropped", func(r *Record) { r.Home = "" }}, |
| + {"home repointed", func(r *Record) { r.Home = "other.go.tito.io" }}, |
| + } { |
| + after := replica |
| + c.mut(&after) |
| + if protectedFieldsEqual(replica, after) { |
| + t.Errorf("%s: treated as equal — an old console would silently strip the replica's link to its home", c.name) |
| + } |
| + } |
| + |
| + // The opt-in is the PATCH door's to change, and the guard must let it. |
| + home := Record{Schema: 1, Host: "acme.go.tito.io", Channel: "stable", |
| + Steering: "latency", Replicas: []string{"uk"}} |
| + patched := home |
| + patched.Replicas = []string{"de", "uk"} |
| + if !protectedFieldsEqual(home, patched) { |
| + t.Error("a replicas PATCH was refused by the guard: the console's own door would 409 on every opt-in change") |
| + } |
| +} |