amadan
Merge pull request #409 from carlosframework/edge-replica-placements
provision, edge: replica placements — the routing half of edge checkout (#162)
25 files changed,
+1220
−30
apimode.go+29 −2apimode_test.go+3 −3canary.go+1 −1cli.go+12 −0deploy.go+1 −1docs/site/cli.md+14 −0instances_cli.go+27 −1instances_environment_test.go+2 −2instances_privatestorage_test.go+1 −1internal/console/apiinstances.go+104 −1internal/console/apiinstances_test.go+78 −0internal/edgesfile/edgesfile.go+3 −2internal/host/edgesfile.go+32 −1internal/host/edgesfile_test.go+103 −0internal/provision/provision.go+150 −1internal/provision/provision_test.go+89 −0internal/provision/reconcile.go+65 −0internal/provision/reconcile_test.go+165 −0internal/provision/replica.go+105 −0internal/registry/registry.go+110 −6internal/registry/registry_test.go+25 −0internal/steering/deployment_test.go+1 −1internal/steering/steering.go+33 −2internal/steering/steering_test.go+66 −4mutation_message_test.go+1 −1
CI passed — run details
diff --git a/apimode.go b/apimode.go| index a1d9574d8..d68941126 100644 |
| --- a/apimode.go |
| +++ b/apimode.go |
| @@ -2937,6 +2937,31 @@ type instanceProbeReq struct { |
| Status int `json:"status,omitempty"` |
| } |
| +// instanceReplicaReq is the #162 replica slice of an instances-create body: |
| +// the role ("" or "replica") and, for a home record, the account's replica |
| +// placement set. role() keeps the wire clean for the overwhelmingly common |
| +// no-replica create: a nil request sends neither key, byte-identical to |
| +// every body before the field existed. |
| +type instanceReplicaReq struct { |
| + Role string |
| + Set []string |
| +} |
| + |
| +func (rq *instanceReplicaReq) role() string { |
| + if rq == nil { |
| + return "" |
| + } |
| + return rq.Role |
| +} |
| + |
| +// set returns the placement set, nil-safe — a nil request sends no key. |
| +func (rq *instanceReplicaReq) set() []string { |
| + if rq == nil { |
| + return nil |
| + } |
| + return rq.Set |
| +} |
| + |
| // instanceCreate asks the console to create one instance record — the |
| // create-only write whose materialization (provision.Reconcile, on the box) |
| // is what brings the host live. The console stamps pool from the manifest |
| @@ -2949,7 +2974,7 @@ type instanceProbeReq struct { |
| // upstream (kind defaults to "instance", which needs no addr at all), and |
| // probe nil means unprobed — the same "absent, not zero" contract the wire |
| // carries for every optional field on this door. |
| -func (s *apiSession) instanceCreate(ctx context.Context, a apiAcct, app, host, region, channel, kind, environment string, addrs []string, probe *instanceProbeReq, messages ...string) (apiInstanceRecord, error) { |
| +func (s *apiSession) instanceCreate(ctx context.Context, a apiAcct, app, host, region, channel, kind, environment string, addrs []string, probe *instanceProbeReq, replicas *instanceReplicaReq, messages ...string) (apiInstanceRecord, error) { |
| message := firstMutationMessage(messages) |
| in := struct { |
| Host string `json:"host"` |
| @@ -2959,8 +2984,10 @@ func (s *apiSession) instanceCreate(ctx context.Context, a apiAcct, app, host, r |
| Environment string `json:"environment,omitempty"` |
| Addrs []string `json:"addrs,omitempty"` |
| Probe *instanceProbeReq `json:"probe,omitempty"` |
| + Role string `json:"role,omitempty"` |
| + Replicas []string `json:"replicas,omitempty"` |
| Message string `json:"message,omitempty"` |
| - }{host, region, channel, kind, environment, addrs, probe, message} |
| + }{host, region, channel, kind, environment, addrs, probe, replicas.role(), replicas.set(), message} |
| status, body, err := s.postJSON(ctx, apiPath(a, app, "/instances"), in) |
| if err != nil { |
| return apiInstanceRecord{}, err |
diff --git a/apimode_test.go b/apimode_test.go| index 681094f26..7357bcf8d 100644 |
| --- a/apimode_test.go |
| +++ b/apimode_test.go |
| @@ -772,7 +772,7 @@ func TestInstanceCreateSendsAddrsAndProbe(t *testing.T) { |
| s := newTestSession(ts.URL) |
| rec, err := s.instanceCreate(context.Background(), apiAcct{Seg: "bah"}, "app", |
| "a.example.com", "", "", "", "", []string{"10.0.0.1:8090", "10.0.0.2:8090"}, |
| - &instanceProbeReq{Path: "/healthz", Status: 200}) |
| + &instanceProbeReq{Path: "/healthz", Status: 200}, nil) |
| if err != nil { |
| t.Fatalf("instanceCreate: %v", err) |
| } |
| @@ -799,7 +799,7 @@ func TestInstanceCreateSendsAddrsAndProbe(t *testing.T) { |
| // No addrs/probe given at all: neither key crosses the wire. |
| gotBody = "" |
| if _, err := s.instanceCreate(context.Background(), apiAcct{Seg: "bah"}, "app", |
| - "b.example.com", "", "", "", "", nil, nil); err != nil { |
| + "b.example.com", "", "", "", "", nil, nil, nil); err != nil { |
| t.Fatalf("instanceCreate (no addrs/probe): %v", err) |
| } |
| if strings.Contains(gotBody, "addrs") || strings.Contains(gotBody, "probe") { |
| @@ -3766,7 +3766,7 @@ func TestAPISessionInstanceCreateSendsKind(t *testing.T) { |
| defer ts.Close() |
| rec, err := newTestSession(ts.URL).instanceCreate(context.Background(), |
| - apiAcct{Seg: "bab"}, "mysite", "mysite.bab.oncarlos.test", "", "edge", tc.kind, "", nil, nil) |
| + apiAcct{Seg: "bab"}, "mysite", "mysite.bab.oncarlos.test", "", "edge", tc.kind, "", nil, nil, nil) |
| if err != nil { |
| t.Fatalf("instanceCreate: %v", err) |
| } |
diff --git a/canary.go b/canary.go| index da7f63e9c..e295f95f5 100644 |
| --- a/canary.go |
| +++ b/canary.go |
| @@ -404,7 +404,7 @@ func runCanaryDeploy(args []string) { |
| pal := colorFor(stdout) |
| switch canaryRecordPlan(recs, host, channel) { |
| case canaryPlanCreate: |
| - rec, cerr := sess.instanceCreate(ctx, acct, app2, host, "", channel, recKind, env, nil, nil) |
| + rec, cerr := sess.instanceCreate(ctx, acct, app2, host, "", channel, recKind, env, nil, nil, nil) |
| if cerr != nil { |
| log.Fatalf("canary: create %s: %v", host, cerr) |
| } |
diff --git a/cli.go b/cli.go| index f6e7dceb3..ad82f5462 100644 |
| --- a/cli.go |
| +++ b/cli.go |
| @@ -239,6 +239,18 @@ func runAdd(args []string) { |
| // steering opt-in the moment the steering housekeeper duty (Task 6) |
| // starts converging DNS from it. Preserve it the same way. |
| route.Steering = existing.Steering |
| + // Replicas (#162) has the same shape and a different preserve rule: |
| + // the column CONVERGES from the record in both directions (the one |
| + // deliberate widening), so the next converge tick repairs a re-add's |
| + // blank anyway — but between this write and that tick the edges file |
| + // would publish the account without its replica placements, a window |
| + // of residence lying for seconds-to-minutes. Preserve it here so the |
| + // window does not exist. The replica MARK is deliberately NOT |
| + // preserved: it rides the record's Role, which this verb cannot |
| + // express, and a re-add minting an unmarked row over a replica's is |
| + // a shape change the next converge repairs — the mark is a routing |
| + // annotation, not a loss of state. |
| + route.Replicas = existing.Replicas |
| } |
| if err := r.Upsert(route); err != nil { |
| log.Fatal(err) |
diff --git a/deploy.go b/deploy.go| index 67079baec..38874a9ce 100644 |
| --- a/deploy.go |
| +++ b/deploy.go |
| @@ -855,7 +855,7 @@ func mintPlatformHost(ctx context.Context, sess *apiSession, acct apiAcct, app, |
| // No environment: the app's apex serves its default config bundle, which |
| // is exactly right — this is the app's primary host, the one every other |
| // bundle's origin values are written to name. |
| - rec, err := sess.instanceCreate(ctx, acct, app, app+"."+m.Domain, "", channel, "static", "", nil, nil, firstMutationMessage(messages)) |
| + rec, err := sess.instanceCreate(ctx, acct, app, app+"."+m.Domain, "", channel, "static", "", nil, nil, nil, firstMutationMessage(messages)) |
| if err != nil { |
| return "", err |
| } |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index e9f387255..f8428bf1e 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -580,6 +580,20 @@ bind it here, then restart the instance to pick the values up. |
| carlos instances set-environment --app jam --host new-queue.jam.bab.oncarlos.com --environment jam-canary |
| ``` |
| +An instance can also be an account's **replica**: a second instance of the |
| +same app, holding a one-way mirror of the live instance's data, placed near |
| +the buyers so a sale does not wait on the home region. |
| +`carlos instances create --app jam --host checkout-uk.jam.bab.oncarlos.com |
| +--region uk --role replica` provisions one. It needs `--region`: the replica |
| +lives where you place it, and only the box carrying that placement mints it. |
| +The live instance's own record names where replicas are wanted — |
| +`--replicas ie,uk` on its create — and that is what publishes the placement |
| +to the edge: the account's public hosts become servable in those regions, |
| +with DNS latency routing still pointing buyers at the home. A replica never |
| +holds the account's authoritative data, and until a batch of orders drains |
| +home it holds order data in its own region — so the placement set is an |
| +explicit decision on the home's record, never a default. |
| + |
| #### Storage of its own, for every instance |
| `--private-storage` on `enable` is one declaration that changes what every |
diff --git a/instances_cli.go b/instances_cli.go| index fe631ed67..6aaf247a6 100644 |
| --- a/instances_cli.go |
| +++ b/instances_cli.go |
| @@ -300,6 +300,8 @@ func runInstancesCreate(args []string) { |
| consoleFlag := consoleOption(fs, "console to create through (default: the only logged-in one)") |
| acctRaw := fs.String("account", "", "account (sqid or name) the app belongs to (default: your only account, or the configured/chosen one)") |
| acctIDRaw := fs.String("account-id", "", "account sqid, matched as a sqid ONLY — supersedes --account") |
| + role := fs.String("role", "", "instance role (default: the serving home); \"replica\" provisions a second instance of the account holding a one-way mirror of it, selling near the buyer — needs --region naming the replica's placement") |
| + replicas := fs.String("replicas", "", "comma-separated placement codes the account's replicas live in (the HOME record's opt-in; e.g. ie,uk) — this instance's own replica must be --role replica with --region") |
| fs.Usage = flagUsage(fs, instancesRecordsUsage) |
| fs.Parse(args) |
| if *host == "" { |
| @@ -319,7 +321,8 @@ func runInstancesCreate(args []string) { |
| } |
| rec, err := sess.instanceCreate(context.Background(), acct, app2, |
| strings.TrimSpace(*host), strings.TrimSpace(*region), strings.TrimSpace(*channel), |
| - strings.TrimSpace(*kind), strings.TrimSpace(*environment), addrs, probe, message) |
| + strings.TrimSpace(*kind), strings.TrimSpace(*environment), addrs, probe, |
| + replicaReqFromFlags(*role, *replicas, set["region"]), message) |
| if err != nil { |
| log.Fatalf("instances create: %v", err) |
| } |
| @@ -331,6 +334,29 @@ func runInstancesCreate(args []string) { |
| mutedf(stdout, " the host comes live on the next reconcile pass on its box") |
| } |
| +// replicaReqFromFlags builds the create body's replica slice from the CLI's |
| +// --role/--replicas pair, or nil when neither flag was given — a nil request |
| +// sends neither key, byte-identical to every create before #162 existed. |
| +// The refusals mirror the console door's (checkReplicaCreate): a replica's |
| +// placement IS its --region, the placement set is the HOME record's opt-in |
| +// and rides a PLAIN create, and a replica never carries a set of its own. |
| +// Refusing here names what is missing without a console round trip. |
| +func replicaReqFromFlags(role, replicas string, regionGiven bool) *instanceReplicaReq { |
| + if role == "" && replicas == "" { |
| + return nil |
| + } |
| + if role != "" && replicas != "" { |
| + log.Fatal("instances create: --role replica with --replicas: the placement set is the HOME record's opt-in — a replica does not carry one (drop --replicas, or drop --role)") |
| + } |
| + if role == "" { |
| + return &instanceReplicaReq{Set: parseAddrList(replicas, true)} |
| + } |
| + if !regionGiven { |
| + log.Fatal("instances create: --role replica needs --region naming the replica's placement") |
| + } |
| + return &instanceReplicaReq{Role: role} |
| +} |
| + |
| // parseAddrList splits a comma-separated --addr/--addrs flag into the wire's |
| // []string, trimming whitespace and dropping empty elements the same way |
| // --regions' comma-split does (instances_cli.go's runInstancesEnable). given |
diff --git a/instances_environment_test.go b/instances_environment_test.go| index ef65b0daf..ce60e627b 100644 |
| --- a/instances_environment_test.go |
| +++ b/instances_environment_test.go |
| @@ -27,7 +27,7 @@ func TestInstanceCreateSendsEnvironmentOnlyWhenGiven(t *testing.T) { |
| s := newTestSession(ts.URL) |
| rec, err := s.instanceCreate(context.Background(), apiAcct{Seg: "bah"}, "app", |
| - "canary.example.com", "", "canary/x", "", "jam-canary", nil, nil) |
| + "canary.example.com", "", "canary/x", "", "jam-canary", nil, nil, nil) |
| if err != nil { |
| t.Fatalf("instanceCreate: %v", err) |
| } |
| @@ -43,7 +43,7 @@ func TestInstanceCreateSendsEnvironmentOnlyWhenGiven(t *testing.T) { |
| gotBody = "" |
| if _, err := s.instanceCreate(context.Background(), apiAcct{Seg: "bah"}, "app", |
| - "plain.example.com", "", "", "", "", nil, nil); err != nil { |
| + "plain.example.com", "", "", "", "", nil, nil, nil); err != nil { |
| t.Fatalf("instanceCreate (no environment): %v", err) |
| } |
| if strings.Contains(gotBody, "environment") { |
diff --git a/instances_privatestorage_test.go b/instances_privatestorage_test.go| index 1531196fe..1572b5550 100644 |
| --- a/instances_privatestorage_test.go |
| +++ b/instances_privatestorage_test.go |
| @@ -73,7 +73,7 @@ func TestInstanceCreateDecodesThePrivateStoreBucket(t *testing.T) { |
| s := newTestSession(ts.URL) |
| rec, err := s.instanceCreate(context.Background(), apiAcct{Seg: "bah"}, "jam", |
| - "acme.jam.example.com", "", "", "", "", nil, nil) |
| + "acme.jam.example.com", "", "", "", "", nil, nil, nil) |
| if err != nil { |
| t.Fatalf("instanceCreate: %v", err) |
| } |
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go| index 48e87f5d4..f63f73e21 100644 |
| --- a/internal/console/apiinstances.go |
| +++ b/internal/console/apiinstances.go |
| @@ -548,6 +548,58 @@ func recordView(rec provision.Record) apiRecordView { |
| return apiRecordView{Record: rec, Gated: rec.Gate != ""} |
| } |
| +// replicaFields is the #162 slice of the instance-create body. Role and |
| +// placement travel together or not at all — "a replica without a placement" |
| +// and "a placement without a replica role" are both half a decision, and |
| +// each half alone produces a record some other box will act on while the |
| +// writer thinks they asked for nothing. Replicas (the home's own placement |
| +// set) rides the same body so one call can provision the replica AND publish |
| +// where the account's replicas live. |
| +type replicaFields struct { |
| + Role string `json:"role,omitempty"` |
| + Replicas []string `json:"replicas,omitempty"` |
| +} |
| + |
| +// checkReplicaCreate is the create door's replica rails (#162). Returns nil |
| +// when the request carries no replica fields at all — the overwhelmingly |
| +// common shape, byte-identical to every create before this door existed — |
| +// and a refusal when it carries half of one. The home's own placement set is |
| +// canonicalized (sorted, deduped, ValidRegion-checked) in place, so the wire |
| +// column never carries a non-canonical order and the reconciler's converge |
| +// has nothing to re-canonicalize on the next pass. |
| +func checkReplicaCreate(req replicaFields, rec *provision.Record) *doorRefusal { |
| + if req.Role == "" && len(req.Replicas) == 0 { |
| + return nil |
| + } |
| + // A replica's placement IS its region — one placement, the create's own |
| + // --region — so an empty one is half a replica: a mirror with nowhere to |
| + // live. (The armed-placement gate below refuses one that names no real |
| + // box; this refuses one that names nothing at all.) |
| + if req.Role != "" && rec.Region == "" { |
| + return &doorRefusal{http.StatusBadRequest, |
| + "role replica needs a placement — pass the placement code the replica lives in as the create's region"} |
| + } |
| + // A replica does not carry a replica set of its own: the set is the |
| + // HOME's opt-in naming where its replicas live, and a replica of a |
| + // replica is not a shape the platform carries. |
| + if req.Role != "" && len(req.Replicas) > 0 { |
| + return &doorRefusal{http.StatusBadRequest, |
| + "role replica with replicas: the placement set is the HOME record's opt-in — a replica does not carry one"} |
| + } |
| + // The home's own placement set is canonicalized (sorted, deduped, |
| + // ValidRegion-checked) in place, so the wire column never carries a |
| + // non-canonical order and the reconciler's converge has nothing to |
| + // re-canonicalize on the next pass. |
| + if len(req.Replicas) > 0 { |
| + norm, err := provision.ValidReplicas(req.Replicas) |
| + if err != nil { |
| + return &doorRefusal{http.StatusBadRequest, err.Error()} |
| + } |
| + rec.Replicas = norm |
| + } |
| + return nil |
| +} |
| + |
| // handleAPIInstanceCreate is the point of the feature: one create-only |
| // write to the bucket, in spec §7's refusal order — gate, budget, manifest, |
| // host, region, channel, write. Every field the record carries that policy |
| @@ -621,6 +673,10 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| Environment string `json:"environment"` |
| Addrs []string `json:"addrs"` |
| Health *provision.Health `json:"probe"` |
| + // The replica fields (#162), embedded so their wire keys sit beside |
| + // kind/environment's — one flat body, no nesting for the caller to |
| + // discover. |
| + replicaFields |
| } |
| if !readJSON(w, r, &req) { |
| return |
| @@ -714,7 +770,31 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| // validation (issue #346): pickers speak the code, records keep the |
| // unconstrained value reconcile has always scoped by. |
| region := s.homeRegionIn(strings.TrimSpace(req.Region)) |
| + // The replica rails (#162) run BEFORE the region allow-list, because |
| + // half a decision is a different refusal from "wrong region": a role |
| + // with no placement, or a set on a replica, is a shape error the |
| + // caller can fix from the message alone, and the record must not |
| + // carry either into the shape validation below. The rails also |
| + // canonicalize the placement set, so the record ValidateShape sees — |
| + // and the bucket receives — is the canonical form. |
| + probeRec := provision.Record{Kind: strings.TrimSpace(req.Kind), Region: region, Role: strings.TrimSpace(req.Role)} |
| + if ref := checkReplicaCreate(req.replicaFields, &probeRec); ref != nil { |
| + jsonError(w, ref.Status, ref.Msg) |
| + return |
| + } |
| if placed != "" { |
| + // A fleet-placed app governs every instance's Region uniformly, and |
| + // the armed/hosting gate below is skipped for it — the fleet is the |
| + // placement, validated by its own rail. A replica under that regime |
| + // would land wherever the fleet says, unchecked by anything that |
| + // knows what a replica placement means (#162): refused, not guessed. |
| + // Delete the app's placement first, or place the replica's account |
| + // on a fleet that spans the placement you want. |
| + if probeRec.Role == "replica" { |
| + jsonError(w, http.StatusBadRequest, |
| + "role replica on a fleet-placed app: the app's fleet placement ("+placed+") governs every instance's region, and a replica's placement must be an armed, hosting platform region — remove the app's placement first") |
| + return |
| + } |
| region = placed |
| } else if !provision.ValidRegion(region) || !instanceRegionAllowed(m.Regions, region) { |
| msg := "invalid region" |
| @@ -745,8 +825,25 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| // provision.Record's own (DerivesPaths), asked of a probe record rather |
| // than restated here, so it cannot drift from what reconcile will mint. |
| if placed == "" && region != "" { |
| - probe := provision.Record{Kind: strings.TrimSpace(req.Kind), Addrs: req.Addrs} |
| + probe := provision.Record{Kind: strings.TrimSpace(req.Kind), Addrs: req.Addrs, Role: probeRec.Role} |
| caps, published := placements(ctx, s.cfg.Releases) |
| + // The replica placement rides the same gate as the record's own |
| + // region (#162): a replica named into a placement no box announces |
| + // is a mirror nothing will ever provision — an order ledger in a |
| + // region that does not exist. Dormant with the rest of the gate |
| + // until a box publishes a placement code. |
| + if published && probe.Role == "replica" { |
| + switch { |
| + case !caps[region].Reachable: |
| + jsonError(w, http.StatusBadRequest, |
| + "replica placement "+region+" is not armed in this deployment — armed: "+armedCodes(reachableSet(caps))) |
| + return |
| + case !caps[region].Hosting: |
| + jsonError(w, http.StatusBadRequest, |
| + "replica placement "+region+" is reachable but cannot host an instance — no box there has its hibernation activator configured. Regions that can: "+armedCodes(hostingSet(caps))) |
| + return |
| + } |
| + } |
| switch { |
| case !published: |
| // Dormant: the fleet predates the placement field. |
| @@ -795,6 +892,12 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request) |
| Addrs: req.Addrs, |
| Health: req.Health, |
| } |
| + // The replica rails already ran above (before the region allow-list), |
| + // on a probe record; the canonicalized answers carry onto the real one |
| + // here. The role lands only as "replica" — checkReplicaCreate refused |
| + // everything else before this point. |
| + rec.Role = probeRec.Role |
| + rec.Replicas = probeRec.Replicas |
| // A static platform host is HA by default. Static is the one serving |
| // class every platform box materializes unconditionally |
| // (provision.everywhereClass): no unit, no process, content read from |
diff --git a/internal/console/apiinstances_test.go b/internal/console/apiinstances_test.go| index 2c02e9063..c8e15630e 100644 |
| --- a/internal/console/apiinstances_test.go |
| +++ b/internal/console/apiinstances_test.go |
| @@ -2605,3 +2605,81 @@ func TestAPIInstanceListDBBytes(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// TestAPIInstanceCreateReplicaFields pins the create door's replica rails |
| +// (#162): role and placement travel together or not at all; a valid replica |
| +// create lands role on the record, in the bucket, canonicalized; the home's |
| +// placement set is deduped and sorted by the door, not by the writer's |
| +// typing; and a record carrying none of the fields writes no replica keys — |
| +// the additivity rule, at the door. |
| +func TestAPIInstanceCreateReplicaFields(t *testing.T) { |
| + fx, token := instanceFixture(t, "code-inst-replica") |
| + m := titoManifest() |
| + m.Regions = []string{"ie", "uk"} // the replica placement must be armed |
| + enableInstances(t, fx, "titogo", m) |
| + u := instancesURL(fx, "bab", "titogo") |
| + |
| + // A replica create: role and region together. |
| + status, _, out := rawReq(t, http.MethodPost, u, token, |
| + `{"host":"acme-uk.go.tito.io","region":"uk","role":"replica"}`) |
| + if status != http.StatusCreated { |
| + t.Fatalf("replica create = %d %s, want 201", status, out) |
| + } |
| + stored, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme-uk.go.tito.io") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if stored.Role != "replica" || stored.Region != "uk" { |
| + t.Fatalf("stored replica record = %+v, want role replica in region uk", stored) |
| + } |
| + |
| + // The home record's placement set: canonicalized by the door. The set |
| + // is the HOME's opt-in, carried on a PLAIN record — role stays empty. |
| + status, _, out = rawReq(t, http.MethodPost, u, token, |
| + `{"host":"acme.go.tito.io","region":"ie","replicas":["uk"," ie ","uk"]}`) |
| + if status != http.StatusCreated { |
| + t.Fatalf("home create = %d %s, want 201", status, out) |
| + } |
| + home, err := provision.GetRecord(context.Background(), fx.rel, "bab", "titogo", "acme.go.tito.io") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := strings.Join(home.Replicas, ","); got != "ie,uk" { |
| + t.Fatalf("stored replica set = %q, want canonical ie,uk", got) |
| + } |
| + |
| + // Half a decision is refused, each way. |
| + for _, c := range []struct{ what, body, want string }{ |
| + {"role without region", `{"host":"half1.go.tito.io","role":"replica"}`, "needs a placement"}, |
| + {"replica with a set", `{"host":"half2.go.tito.io","role":"replica","region":"uk","replicas":["uk"]}`, "HOME record's opt-in"}, |
| + {"bad placement", `{"host":"half3.go.tito.io","replicas":["fleet/pi"]}`, "placement code"}, |
| + } { |
| + status, _, out := rawReq(t, http.MethodPost, u, token, c.body) |
| + if status != http.StatusBadRequest { |
| + t.Errorf("%s = %d %s, want 400", c.what, status, out) |
| + continue |
| + } |
| + if !strings.Contains(out, c.want) { |
| + t.Errorf("%s refusal = %s, want it to name %q", c.what, out, c.want) |
| + } |
| + } |
| + keys, _ := fx.rel.List(context.Background(), provision.RecordsPrefix("bab", "titogo")) |
| + if len(keys) != 2 { |
| + t.Fatalf("refused creates left records: %v", keys) |
| + } |
| + |
| + // A shapeless create writes no replica keys. |
| + if status, _, out := rawReq(t, http.MethodPost, u, token, |
| + `{"host":"plain.go.tito.io","region":"ie"}`); status != http.StatusCreated { |
| + t.Fatalf("plain create = %d %s, want 201", status, out) |
| + } |
| + plain, _, err := fx.rel.Get(context.Background(), provision.RecordKey("bab", "titogo", "plain.go.tito.io")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, k := range []string{`"role"`, `"replicas"`, `"replica"`} { |
| + if strings.Contains(string(plain), k) { |
| + t.Errorf("a shapeless create wrote %s into the record: %s", k, plain) |
| + } |
| + } |
| +} |
diff --git a/internal/edgesfile/edgesfile.go b/internal/edgesfile/edgesfile.go| index 15623ecbd..10346696d 100644 |
| --- a/internal/edgesfile/edgesfile.go |
| +++ b/internal/edgesfile/edgesfile.go |
| @@ -17,8 +17,9 @@ |
| // Two readers deliberately keep their own narrow decoders and are NOT |
| // migrated: internal/edge/homemap.go (the serving hot path, with its own |
| // polling and staleness machinery) and internal/steering (which reads no |
| -// Hosts at all — steering is any-edge-any-app). Both are named here so the |
| -// consumer list stays complete rather than merely short. |
| +// role values — since #162 it does read the per-host replica mark, but its |
| +// decode stays its own for the same isolation reason). Both are named here |
| +// so the consumer list stays complete rather than merely short. |
| package edgesfile |
| import ( |
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go| index 2c9c666dd..40a04a93d 100644 |
| --- a/internal/host/edgesfile.go |
| +++ b/internal/host/edgesfile.go |
| @@ -51,6 +51,19 @@ type edgeHost struct { |
| // route beat an alias on a name collision across boxes: a tenant |
| // attaching an operator's hostname must never capture it fleet-wide. |
| Alias bool `json:"alias,omitempty"` |
| + // Replica marks the account's checkout replica for this host (#162): |
| + // this box serves the account from a one-way mirror of it — same app, |
| + // same account, role=replica record — instead of holding the |
| + // authoritative data. Additive (omitempty), so every file written |
| + // before the field existed parses byte-identically, and a reader that |
| + // has never heard of it ignores it. Published from the HOME route's |
| + // Replicas column (registry.Route.Replicas): a replica placement is |
| + // where the account is SERVED, and the home's row is the one that |
| + // knows. Read by the steering converger (a replica placement earns a |
| + // latency record) and available to every edges-file reader; the edge's |
| + // own dispatch never acts on it — a replica box serves its row exactly |
| + // as any home serves its own, and a replica never hairpins. |
| + Replica bool `json:"replica,omitempty"` |
| // Wildcard marks a delegated customer wildcard parent (wildcard_domains, |
| // #305): the key is "*.<parent>" and every edge should load the parent's |
| // mirrored wildcard PEM. With role "home" the box also serves the |
| @@ -216,7 +229,13 @@ func edgesFilePass(ctx context.Context, s release.Store, reg *registry.Registry, |
| // answers must apply edge.go's lookupUpstream one-label wildcard |
| // rule itself: an exact match wins, and on a miss "*.<parent>" |
| // (the leading dot of the first label only) is tried next. |
| - hosts[r.Host] = edgeHost{Role: "home", Platform: r.Platform || r.Provisioned} |
| + // |
| + // The replica mark rides the ROW (#162): a route minted from a |
| + // role=replica record publishes with replica:true wherever its own |
| + // host appears, so every reader of the file can tell the account's |
| + // homes from its checkout replicas. See the replica loop below for |
| + // the account apex that rides along. |
| + hosts[r.Host] = edgeHost{Role: "home", Platform: r.Platform || r.Provisioned, Replica: r.Replica} |
| } |
| // Attached custom domains (#151) live in route_aliases, not routes, and |
| // were absent from this file (#287): a non-home edge neither backhauled |
| @@ -269,6 +288,18 @@ func edgesFilePass(ctx context.Context, s release.Store, reg *registry.Registry, |
| hosts[a.Alias] = edgeHost{Role: "home", Alias: true} |
| } |
| + // Replica placements (#162): a row minted from a role=replica record — |
| + // this box holds it BECAUSE its placement is this box's (the reconciler's |
| + // scope test is the only mint gate) — publishes with the replica mark |
| + // set, which the main loop above already stamps onto its own host (the |
| + // mark rides the ROW). Nothing beyond that host is claimed: the home's |
| + // hostname and the platform apex live on the home box's rows, and a name |
| + // this box cannot serve from a local row must not be claimed (a |
| + // replica-claimed apex would let foldHomes' first-sorted-box rule hand |
| + // the apex DIAL target to a replica box fleet-wide). Custom domains |
| + // reach the edges by their owner's CNAME to edge.<apps-domain> anyway |
| + // (the steering fold's own documented path). |
| + |
| // Delegated customer wildcards (#305): the wildcard_domains rows are the |
| // parents this box obtains "*.<parent>" for (a claim with the Wildcard |
| // capability, primary homed here). Published as the catch-all key with |
diff --git a/internal/host/edgesfile_test.go b/internal/host/edgesfile_test.go| index 31c61feec..ad3eaec5a 100644 |
| --- a/internal/host/edgesfile_test.go |
| +++ b/internal/host/edgesfile_test.go |
| @@ -986,3 +986,106 @@ func TestEdgesFileRawWireBodyDataBucketFields(t *testing.T) { |
| t.Fatalf("unset hibernate env still published a data_bucket key: %s", got2) |
| } |
| } |
| + |
| +// --- #162: the replica placement's edges-file entries ---------------------- |
| + |
| +// TestEdgesFilePublishesReplicaEntries pins the data plane of replica |
| +// placements: a served row minted from a role=replica record — the Replica |
| +// mark set, Account/App carried so the account apex derives — publishes its |
| +// own host and the account's platform apex under role "home" with the |
| +// replica mark set. A PLAIN row never gains the mark, whatever placement the |
| +// box publishes; the mark is the row's, not the box's. (Raw map decode, not |
| +// the struct: the test is about the WIRE, and a struct field would make an |
| +// omitempty typo invisible.) |
| +func TestEdgesFilePublishesReplicaEntries(t *testing.T) { |
| + s, reg := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "acme-uk.go.tito.io", Slug: "acme-uk", Kind: "service", Addr: "10.0.0.1:3000", |
| + Status: "ready", Account: "bab", App: "tito", Region: "uk", |
| + Replica: true, |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + // The home row, on the same box only for the test's convenience — its |
| + // entries must NOT carry the mark even when it sits here. |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "acme.go.tito.io", Slug: "acme", Kind: "service", Addr: "10.0.0.2:3000", |
| + Status: "ready", Account: "bab", App: "tito", Region: "ie", |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + var lastBody []byte |
| + if err := edgesFilePass(context.Background(), s, reg, edgesFile{Box: "ukbox", Placement: "uk"}, "1.2.3.4", &lastBody); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, _, err := s.Get(context.Background(), "control/edges/ukbox.json") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var raw struct { |
| + Hosts map[string]struct { |
| + Role string `json:"role"` |
| + Replica bool `json:"replica"` |
| + } `json:"hosts"` |
| + } |
| + if err := json.Unmarshal(got, &raw); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, name := range []string{"acme-uk.go.tito.io"} { |
| + h, ok := raw.Hosts[name] |
| + if !ok { |
| + t.Errorf("replica row published nothing at %s (hosts: %+v)", name, raw.Hosts) |
| + continue |
| + } |
| + if h.Role != "home" || !h.Replica { |
| + t.Errorf("%s = %+v, want role home with the replica mark", name, h) |
| + } |
| + } |
| + if h := raw.Hosts["acme.go.tito.io"]; h.Replica { |
| + t.Errorf("the home row's entry carries the replica mark: %+v — the mark is the ROW's, and the home row is not a replica", h) |
| + } |
| + // The account apex is deliberately NOT published: a name the replica box |
| + // has no local row for must not be claimed (an apex key here would let |
| + // foldHomes' first-sorted-box rule hand the apex dial target to a |
| + // replica box fleet-wide). |
| + if _, ok := raw.Hosts["tito.bab"]; ok { |
| + t.Errorf("replica row published the account apex tito.bab — a name this box has no local row for must not be claimed (hosts: %+v)", raw.Hosts) |
| + } |
| +} |
| + |
| +// TestEdgesFileReplicaMarkFollowsTheRow pins the fail-closed default: a |
| +// plain served row publishes exactly as before #162 — no replica key at all |
| +// on the wire — on a box with no placement published, with no placement, or |
| +// with any placement. The mark cannot leak from the box's identity into an |
| +// unrelated row. |
| +func TestEdgesFileReplicaMarkFollowsTheRow(t *testing.T) { |
| + s, reg := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| + if err := reg.Upsert(registry.Route{ |
| + Host: "acme.go.tito.io", Kind: "service", Addr: "10.0.0.1:3000", |
| + Status: "ready", |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, ident := range []edgesFile{{Box: "annie"}, {Box: "annie", Placement: "uk"}} { |
| + var lastBody []byte |
| + if err := edgesFilePass(context.Background(), s, reg, ident, "1.2.3.4", &lastBody); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, _, err := s.Get(context.Background(), "control/edges/annie.json") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var raw struct { |
| + Hosts map[string]struct { |
| + Replica bool `json:"replica"` |
| + } `json:"hosts"` |
| + } |
| + if err := json.Unmarshal(got, &raw); err != nil { |
| + t.Fatal(err) |
| + } |
| + if h, ok := raw.Hosts["acme.go.tito.io"]; ok && h.Replica { |
| + t.Errorf("hosts = %+v, want no replica mark on a plain row (box placement %q)", raw.Hosts, ident.Placement) |
| + } |
| + } |
| +} |
diff --git a/internal/provision/provision.go b/internal/provision/provision.go| index e958c2dc8..34be6f41f 100644 |
| --- a/internal/provision/provision.go |
| +++ b/internal/provision/provision.go |
| @@ -14,6 +14,7 @@ import ( |
| "errors" |
| "fmt" |
| "github.com/carlosframework/platform/internal/controlname" |
| + "log" |
| "net" |
| "os" |
| "path/filepath" |
| @@ -130,6 +131,11 @@ type Record struct { |
| // even then only once a deployment arms CARLOS_STEERING_ZONE_ID. |
| Steering string `json:"steering,omitempty"` |
| + // Replica is NOT a separate record field: Role carries the role |
| + // vocabulary alone (ValidRole's door), and every reader derives the |
| + // boolean from it. A second spelling of the same fact is two chances |
| + // to disagree. |
| + // |
| // Gate is the bcrypt hash of the host's shared password ("" = open, |
| // the value every record written before this field existed decodes |
| // to). The ONLY place the secret material lives in the bucket; the |
| @@ -140,6 +146,41 @@ type Record struct { |
| // host. Converged into routes.gate by Reconcile's owned-converge arm |
| // (reconcile.go) and read by the edge on every request. Spec docs/superpowers/specs/2026-09-07-password-gate-design.md. |
| Gate string `json:"gate,omitempty"` |
| + |
| + // --- replica fields (carlos/platform discussion #162, edge checkout) --- |
| + |
| + // Role is what this instance IS within its app: "" (the default every |
| + // record written before this field existed decodes to) is the serving |
| + // home — the instance the account's hosts resolve to, the one whose |
| + // data is authoritative. "replica" is a SECOND instance of the same |
| + // account, holding a one-way mirror of it, placed near the buyers: an |
| + // edge-checkout shard selling against a capacity lease from home and |
| + // draining orders back in batches (titogo edge-checkout design |
| + // record, 2026-09-21). The role is NOT a routing fact by itself — it |
| + // takes effect through Replicas (below) and the reconciler's mint |
| + // gate — and ValidRole is the door. |
| + // |
| + // Written only by the create door that also names the placement; no |
| + // PATCH axis flips a record into or out of the replica role in place |
| + // (delete and re-create is the supported transition), because a role |
| + // flip is a data-plane event — which database is authoritative — not |
| + // a shape edit. |
| + Role string `json:"role,omitempty"` |
| + |
| + // Replicas names the placement codes an account's replicas live in: |
| + // the home record's own opt-in, "ie,uk" style, deduplicated, |
| + // canonically sorted and validated against ValidRegion. Empty on |
| + // every record that predates the field = no replicas anywhere = |
| + // today's routing, byte for byte. It is CONVERGED (Reconcile writes |
| + // the record's value over the row's on every pass, through |
| + // SetReplicas — the one deliberate exception to that arm's |
| + // never-widen rule, because replica placement is the bucket's |
| + // decision to own: a stale placement left behind after the set |
| + // shrank would keep a retired region's replica on the account's |
| + // hosts forever), published by the edges file, honoured by the |
| + // steering converger — and never read by the edge's own dispatch, |
| + // which never hairpins a replica. |
| + Replicas []string `json:"replicas,omitempty"` |
| } |
| // Health is the per-route probe config (HA spec Part II). Zero value = |
| @@ -387,6 +428,59 @@ func ValidSteering(s string) bool { |
| return false |
| } |
| +// ValidRole pins the record's role vocabulary (#162): "" (the serving |
| +// home — every record written before the field existed decodes to it, |
| +// which is what makes the field additive) or "replica" (a one-way-mirror |
| +// instance of the same account, selling near the buyer). Case-sensitive |
| +// and closed, ValidSteering's discipline: a value a future reader has |
| +// never heard of is refused where it is typed rather than discovered as a |
| +// row nothing routes for. |
| +func ValidRole(s string) bool { |
| + switch s { |
| + case "", "replica": |
| + return true |
| + } |
| + return false |
| +} |
| + |
| +// MaxReplicas caps the placement set one record may name. See validReplicas. |
| +const MaxReplicas = 8 |
| + |
| +// ValidReplicas normalizes and validates a replica placement set: trim, |
| +// drop empties, dedupe, sort (the canonical order every comma-joined |
| +// column here keeps), and check each element against ValidRegion — the |
| +// same vocabulary a record's own Region speaks, so "is this placement |
| +// armed" stays one question with one answer. Exported because the console's |
| +// create door canonicalizes the request body through the SAME derivation |
| +// the reconciler re-checks at (neither side trusts the other about what is |
| +// in the bucket). The count cap is the residence rule made concrete |
| +// (design record § Residence): replicas are per-account opt-in constrained |
| +// by the account's residence policy, not "sprinkle the fleet" — a number a |
| +// deployment can raise deliberately beats an unbounded set nobody sized. |
| +func ValidReplicas(regions []string) ([]string, error) { |
| + if len(regions) > MaxReplicas { |
| + return nil, fmt.Errorf("replicas: %d placements, at most %d — replicas hold order data in every region named until a drain confirms, so each one is a residence decision, not a cache hint", len(regions), MaxReplicas) |
| + } |
| + seen := make(map[string]bool, len(regions)) |
| + out := make([]string, 0, len(regions)) |
| + for _, r := range regions { |
| + r = strings.TrimSpace(r) |
| + if r == "" { |
| + continue |
| + } |
| + if !ValidRegion(r) { |
| + return nil, fmt.Errorf("replica placement %q: want a placement code (e.g. ie, uk)", r) |
| + } |
| + if seen[r] { |
| + continue |
| + } |
| + seen[r] = true |
| + out = append(out, r) |
| + } |
| + sort.Strings(out) |
| + return out, nil |
| +} |
| + |
| // MaxUpstreams caps the Addrs set one record may name. The prober probes a |
| // host's upstreams SEQUENTIALLY once per interval (internal/edge/prober.go), |
| // and every scale figure the multi-upstream design rests on — probe traffic, |
| @@ -405,9 +499,34 @@ func ValidateShape(rec Record) error { |
| if !ValidKind(rec.Kind) { |
| return fmt.Errorf("kind %q: want instance, service or static (empty means instance)", rec.Kind) |
| } |
| + if !ValidRole(rec.Role) { |
| + return fmt.Errorf("role %q: want \"\" or \"replica\"", rec.Role) |
| + } |
| if !ValidSteering(rec.Steering) { |
| return fmt.Errorf("steering %q: want \"\" or \"latency\"", rec.Steering) |
| } |
| + // A replica is a process holding a mirror of live data — never a file |
| + // tree, never a foreign upstream the platform did not spawn. Pinning |
| + // the class here (not only at the reconciler's mint gate) keeps the rail |
| + // holding no matter which writer produced the record — and the ADDR half |
| + // matters as much as the kind half: a replica with a public addr would |
| + // pass everywhereClass's dial test and mint on EVERY non-fleet box, the |
| + // residence rule's second door. |
| + if rec.Role == "replica" && rec.KindOrDefault() != "instance" { |
| + return errors.New("role replica with kind " + rec.KindOrDefault() + ": a replica is an instance of the account — a process the activator spawns holding a mirror of its data — drop kind or drop role") |
| + } |
| + if rec.Role == "replica" && len(rec.AddrsOrDefault()) > 0 { |
| + return errors.New("role replica with addr: a replica's process is the platform's own, spawned by the activator on the box its placement names — an upstream would make it an everywhere-class route and mint it on every box — drop the addr or drop role") |
| + } |
| + for _, r := range rec.Replicas { |
| + r = strings.TrimSpace(r) |
| + if r != "" && !ValidRegion(r) { |
| + return fmt.Errorf("replica placement %q: want a placement code (e.g. ie, uk)", r) |
| + } |
| + } |
| + if len(rec.Replicas) > MaxReplicas { |
| + return fmt.Errorf("replicas: %d placements, at most %d", len(rec.Replicas), MaxReplicas) |
| + } |
| if rec.Gate != "" { |
| if _, err := bcrypt.Cost([]byte(rec.Gate)); err != nil { |
| return errors.New("gate: want a bcrypt hash — the console hashes the password; a record never carries it in clear") |
| @@ -569,7 +688,37 @@ func ValidRecordHost(rec Record, domain string) bool { |
| var regionPoolRe = regexp.MustCompile(`^[a-z0-9-]{0,16}$`) |
| func ValidRegion(region string) bool { return regionPoolRe.MatchString(region) } |
| -func ValidPool(pool string) bool { return regionPoolRe.MatchString(pool) } |
| + |
| +// replicaSet renders a record's replica placement set for the wire column |
| +// that carries it (registry.Route.Replicas): comma-joined placement codes in |
| +// canonical (sorted, deduplicated, ValidRegion-checked at the door) order, |
| +// "" = none. routeFor and convergeOwned both call this — one derivation, |
| +// two writers, the upstreamCols rule: a second, independent rendering is how |
| +// a route minted correctly comes to be "converged" onto a different value |
| +// every pass, forever. |
| +func replicaSet(regions []string) string { |
| + norm, err := ValidReplicas(regions) |
| + if err != nil { |
| + // The door has already checked the set; a record that slipped past |
| + // it (a hand-written bucket body — the reconcile pass reads those) |
| + // still renders DETERMINISTICALLY, never as two different columns |
| + // on two boxes. Sorted, deduped, best-effort — and LOGGED, so a |
| + // hand-edited body is visible rather than silently normalized. |
| + log.Printf("provision: replicas set failed validation (%v) — rendering the sorted, deduplicated remainder", err) |
| + seen := map[string]bool{} |
| + var keep []string |
| + for _, r := range regions { |
| + if r = strings.TrimSpace(r); r != "" && !seen[r] { |
| + seen[r] = true |
| + keep = append(keep, r) |
| + } |
| + } |
| + norm = keep |
| + sort.Strings(norm) |
| + } |
| + return strings.Join(norm, ",") |
| +} |
| +func ValidPool(pool string) bool { return regionPoolRe.MatchString(pool) } |
| // envNameRe is appconfig.ValidEnvironmentName's own shape (pipelines design |
| // §4a), spelled out again here rather than imported: appconfig already |
diff --git a/internal/provision/provision_test.go b/internal/provision/provision_test.go| index e233f5bce..0f6609ea2 100644 |
| --- a/internal/provision/provision_test.go |
| +++ b/internal/provision/provision_test.go |
| @@ -1116,6 +1116,95 @@ func TestLayoutJournalPath(t *testing.T) { |
| } |
| } |
| +// --- #162: replica role and placement set --------------------------------- |
| + |
| +// TestRecordReplicaFieldsAreAdditive extends TestRecordWireShapeIsAdditive |
| +// to the replica pair: a record written before the fields existed reads as |
| +// "the serving home with no replicas anywhere", byte for byte the routing |
| +// and placement truth every box in the fleet held before #162. |
| +func TestRecordReplicaFieldsAreAdditive(t *testing.T) { |
| + const legacy = `{"schema":1,"host":"acme.go.tito.io","slug":"acme","app":"tito",` + |
| + `"account":"bab","pool":"tito","region":"ie","channel":"stable",` + |
| + `"created_at":"2026-08-04T09:00:00Z"}` |
| + var rec Record |
| + if err := json.Unmarshal([]byte(legacy), &rec); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rec.Role != "" { |
| + t.Errorf("Role = %q, want empty", rec.Role) |
| + } |
| + if len(rec.Replicas) != 0 { |
| + t.Errorf("Replicas = %v, want none", rec.Replicas) |
| + } |
| + if got := rec.Replicas; got != nil { |
| + t.Errorf("Replicas = %v on a legacy record, want nil", got) |
| + } |
| +} |
| + |
| +// TestValidRolePinsTheVocabulary: closed and case-sensitive, ValidSteering's |
| +// discipline — a value no reader has heard of is refused where it is typed. |
| +func TestValidRolePinsTheVocabulary(t *testing.T) { |
| + for _, role := range []string{"", "replica"} { |
| + if !ValidRole(role) { |
| + t.Errorf("ValidRole(%q) = false, want true", role) |
| + } |
| + } |
| + for _, role := range []string{"Replica", "REPLICA", "read", "mirror", "home"} { |
| + if ValidRole(role) { |
| + t.Errorf("ValidRole(%q) = true, want false", role) |
| + } |
| + } |
| +} |
| + |
| +// TestValidReplicasCanonicalizesAndRejects: the set is canonical (sorted, |
| +// deduplicated) so the wire column and the reconciler's converge compare |
| +// equal regardless of the order the writer typed; a placement outside the |
| +// Region vocabulary is refused at the door rather than discovered as a |
| +// placement no box ever announces. |
| +func TestValidReplicasCanonicalizesAndRejects(t *testing.T) { |
| + got, err := ValidReplicas([]string{"uk", " ie ", "", "uk", "us"}) |
| + if err != nil { |
| + t.Fatalf("ValidReplicas: %v", err) |
| + } |
| + if strings.Join(got, ",") != "ie,uk,us" { |
| + t.Errorf("ValidReplicas = %v, want [ie uk us] — sorted and deduplicated", got) |
| + } |
| + if _, err := ValidReplicas([]string{"fleet/pi"}); err == nil { |
| + t.Error("ValidReplicas(fleet/pi) accepted — a fleet is a tenancy boundary, not a replica placement") |
| + } |
| + if _, err := ValidReplicas([]string{"not a place"}); err == nil { |
| + t.Error("ValidReplicas(garbage) accepted") |
| + } |
| + var many []string |
| + for i := 0; i < MaxReplicas+1; i++ { |
| + many = append(many, fmt.Sprintf("r%d", i)) |
| + } |
| + if _, err := ValidReplicas(many); err == nil { |
| + t.Errorf("ValidReplicas(%d placements) accepted, want the residence cap refusal", MaxReplicas+1) |
| + } |
| +} |
| + |
| +// TestValidateShapeReplicaRails pins the shape rules the door enforces for |
| +// the replica role: instance-only, and the placement set's vocabulary. |
| +func TestValidateShapeReplicaRails(t *testing.T) { |
| + replica := Record{Role: "replica", Region: "uk"} |
| + if err := ValidateShape(replica); err != nil { |
| + t.Fatalf("ValidateShape(bare replica): %v", err) |
| + } |
| + if err := ValidateShape(Record{Role: "replica", Kind: "static"}); err == nil { |
| + t.Error("ValidateShape(replica static) accepted — a replica is a process holding a mirror, never a file tree") |
| + } |
| + if err := ValidateShape(Record{Role: "replica", Kind: "service", Addr: "10.0.0.1:8090"}); err == nil { |
| + t.Error("ValidateShape(replica service) accepted — the platform spawns the mirror's process") |
| + } |
| + if err := ValidateShape(Record{Replicas: []string{"fleet/pi"}}); err == nil { |
| + t.Error("ValidateShape(fleet replica placement) accepted") |
| + } |
| + if err := ValidateShape(Record{Replicas: []string{"ie", "uk", "ie"}}); err != nil { |
| + t.Errorf("ValidateShape(duplicate placement): %v — deduplication is the door's job, not a refusal", err) |
| + } |
| +} |
| + |
| // PrivateStorage is additive and its ZERO VALUE is the legacy behaviour: a |
| // manifest written before the field existed — every manifest in a |
| // production bucket — must read as an app whose instances share the app's |
diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go| index 7f4b6dd86..8b3d2d9e8 100644 |
| --- a/internal/provision/reconcile.go |
| +++ b/internal/provision/reconcile.go |
| @@ -200,6 +200,16 @@ func productionClear(ctx context.Context, s release.Store, pub ed25519.PublicKey |
| // elsewhere), while one wrongly promoted to everywhere is a box proxying to an |
| // upstream it cannot reach. |
| func everywhereClass(rec Record) bool { |
| + // A replica is the residence rule made structural (#162): it belongs to |
| + // the box its placement names and nowhere else, and this widening is the |
| + // mint gate for every non-fleet box. ValidateShape refuses a replica |
| + // with an upstream, so the addr arm below could never fire on one — but |
| + // "could never through THIS door" is not "cannot", and a replica of a |
| + // static kind would pass the kind arm above if the kind rail ever |
| + // moved. The role check is FIRST and absolute. |
| + if rec.Role == "replica" { |
| + return false |
| + } |
| // Unit names the systemd unit that owns this route's process (issue |
| // #110); adoption CYCLES it. That unit exists on the home box, so a far |
| // box adopting the row would be stamping authority over a unit name that |
| @@ -256,6 +266,12 @@ func everywhereClass(rec Record) bool { |
| // documents as human-gated, which is worse. The conservative gate stands; the |
| // gap is tracked in the bookkeeping issue, not papered over here. |
| func everywhereRoute(rt registry.Route) bool { |
| + // The replica mark is everywhereClass's role check, read off the row |
| + // (the record whose class it was may be exactly what is gone) — the |
| + // same FIRST-and-absolute position, for the same reason. |
| + if rt.Replica { |
| + return false |
| + } |
| if rt.Unit != "" || rt.Hibernate || rt.Backing != "" || rt.Tunnel != "" || rt.Caps != "" { |
| return false |
| } |
| @@ -352,6 +368,8 @@ func routeFor(rec Record, dataDir, seed string) registry.Route { |
| Channel: rec.Channel, Environment: rec.Environment, |
| Hibernate: rec.HibernateOrDefault(), StatusNamed: rec.StatusNamedOrDefault(), |
| Steering: rec.Steering, |
| + Replicas: replicaSet(rec.Replicas), |
| + Replica: rec.Role == "replica", |
| Gate: rec.Gate, |
| Provisioned: true, Seed: seed, Status: "ready", |
| } |
| @@ -497,6 +515,14 @@ func shapeMismatch(existing, want registry.Route) string { |
| return "platform (owned by the domains reconcile pass)" |
| case existing.Canary: |
| return "canary (owned by the domains reconcile pass)" |
| + // Replicas is deliberately absent, replication's twin: the column is |
| + // CONVERGED in the owned arm (SetReplicas — the record is authoritative |
| + // in both directions), so a divergence here is repaired, never a reason |
| + // to refuse adoption. Listing it would opt every route with a settled |
| + // replica set out of the backfill permanently and invisibly, the exact |
| + // trade the replication paragraph below refuses — and a replica set |
| + // grants no authority over the route's process; it names where the |
| + // account's mirrors live. |
| case existing.App != want.App: |
| return fmt.Sprintf("app %q/%q", existing.App, want.App) |
| case existing.Account != want.Account: |
| @@ -626,6 +652,36 @@ func convergeOwned(reg *registry.Registry, existing registry.Route, rec Record, |
| log.Printf("provision: converged %s steering %q -> %q", rec.Host, existing.Steering, rec.Steering) |
| } |
| } |
| + // Replicas CONVERGES, where steering merely repoints: the bucket's value |
| + // is authoritative in BOTH directions, including down to none — a set |
| + // that shrank must actually shrink on the row, or the edges file goes on |
| + // publishing a replica placement the account no longer has. This is the |
| + // one deliberate widening the owned arm permits, and the reason is in |
| + // Record.Replicas' doc comment: replica placement is the bucket's |
| + // decision to own, not an operator column to preserve. |
| + if want := replicaSet(rec.Replicas); existing.Replicas != want { |
| + if dryRun { |
| + log.Printf("provision: dry-run: would converge %s replicas %q -> %q", rec.Host, existing.Replicas, want) |
| + } else if err := reg.SetReplicas(rec.Host, want); err != nil { |
| + log.Printf("provision: converge replicas on %s: %v", rec.Host, err) |
| + } else { |
| + log.Printf("provision: converged %s replicas %q -> %q", rec.Host, existing.Replicas, want) |
| + } |
| + } |
| + // The replica MARK converges the same both-ways way, from the same |
| + // record: it decides how the edges file publishes the row (replica |
| + // mark vs plain home), which is exactly the fact steering's fold sorts |
| + // hosts by. Upsert would preserve it wrong (a re-mint carrying no role |
| + // must DEMOTE a row that carried one), so the narrow setter owns it. |
| + if want := rec.Role == "replica"; existing.Replica != want { |
| + if dryRun { |
| + log.Printf("provision: dry-run: would converge %s replica mark %v -> %v", rec.Host, existing.Replica, want) |
| + } else if err := reg.SetReplica(rec.Host, want); err != nil { |
| + log.Printf("provision: converge replica mark on %s: %v", rec.Host, err) |
| + } else { |
| + log.Printf("provision: converged %s replica mark %v -> %v", rec.Host, existing.Replica, want) |
| + } |
| + } |
| } |
| // Reconcile materializes bucket instance records into registry routes for |
| @@ -731,6 +787,15 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| // Scope. For a home-boxed record this is the region/pool test it |
| // always was; for a static or public-addr one the region half widens |
| // to every platform box in the deployment (everywhereClass, above). |
| + // A replica record narrows it: it belongs ONLY to the box whose |
| + // placement is the record's own Region — the placement the create |
| + // door validated as armed — so a replica record minted on the wrong |
| + // region would mint a route with no mirror behind it and no lease |
| + // speaking to it, a process holding live order data nobody |
| + // authorized into that region (the residence rule, #162). Every |
| + // other box skips it exactly like any other out-of-scope record; |
| + // the replica's own box sees Region == cfg.Region through the |
| + // ordinary test above. |
| if !cfg.inScope(rec.Region, rec.Pool, everywhereClass(rec)) { |
| continue // another agent's jurisdiction, not this pass's business |
| } |
diff --git a/internal/provision/reconcile_test.go b/internal/provision/reconcile_test.go| index e524ad2fb..e877c33dc 100644 |
| --- a/internal/provision/reconcile_test.go |
| +++ b/internal/provision/reconcile_test.go |
| @@ -2735,3 +2735,168 @@ func TestShapeMismatchIgnoresSteering(t *testing.T) { |
| t.Fatal("a genuine kind mismatch was hidden behind the ignored steering column") |
| } |
| } |
| + |
| +// --- #162: replica records, the home's placement set, and the two boxes' -- |
| +// --- division of labour --------------------------------------------------- |
| + |
| +// TestReplicaRecordMintsOnlyOnItsPlacement: a replica record names ONE |
| +// placement — its Region — and only the box carrying that placement mints |
| +// it. Everywhere else the record is out of scope exactly like any other |
| +// region's record: a replica minted on the wrong region would be a process |
| +// holding live order data nobody authorized into that region (the residence |
| +// rule), and a mint on the HOME box would stand up a second process of the |
| +// same account on a box that already holds the authoritative copy. |
| +func TestReplicaRecordMintsOnlyOnItsPlacement(t *testing.T) { |
| + s, sqid := setup(t) |
| + home := acmeRecord(sqid) |
| + if err := CreateRecord(ctx, s, home); err != nil { |
| + t.Fatal(err) |
| + } |
| + replica := Record{Schema: 1, Host: "acme-uk.go.tito.io", Slug: "acme-uk", App: "tito", |
| + Account: sqid, Pool: "tito", Region: "uk", Channel: "stable", |
| + Role: "replica", CreatedAt: time.Now()} |
| + if err := CreateRecord(ctx, s, replica); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + // The uk box mints the replica. |
| + reg := regWith(t) |
| + if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, err := reg.Get("acme-uk.go.tito.io") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if got == nil { |
| + t.Fatal("the uk box did not mint the replica record homed in uk") |
| + } |
| + |
| + // The ie box (the home's region) mints the home and nothing else. |
| + regIE := regWith(t) |
| + if err := Reconcile(ctx, s, regIE, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rt, _ := regIE.Get("acme-uk.go.tito.io"); rt != nil { |
| + t.Fatal("the ie box minted a record whose replica placement is uk — out of scope") |
| + } |
| + if rt, _ := regIE.Get("acme.go.tito.io"); rt == nil { |
| + t.Fatal("the ie box did not mint the home record") |
| + } |
| +} |
| + |
| +// TestHomeRecordConvergesReplicaSet: the home record's placement set is |
| +// authoritative in BOTH directions — the one deliberate widening the owned |
| +// arm permits — because a set that shrank must actually shrink on the row, |
| +// or the edges file goes on publishing a replica placement the account no |
| +// longer has. That is a residence lie, not a cosmetic one. |
| +func TestHomeRecordConvergesReplicaSet(t *testing.T) { |
| + s, sqid := setup(t) |
| + home := acmeRecord(sqid) |
| + home.Replicas = []string{"uk", "ie"} |
| + if err := CreateRecord(ctx, s, home); err != nil { |
| + t.Fatal(err) |
| + } |
| + reg := regWith(t) |
| + if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + if rt, _ := reg.Get("acme.go.tito.io"); rt == nil { |
| + t.Fatal("home not minted") |
| + } else if rt.Replicas != "ie,uk" { |
| + t.Fatalf("minted row replicas = %q, want the canonical sorted set %q", rt.Replicas, "ie,uk") |
| + } |
| + |
| + // The set shrinks in the bucket; the row follows on the next pass. |
| + home.Replicas = []string{"uk"} |
| + if _, err := updateRecord(ctx, s, sqid, "tito", "acme.go.tito.io", func(r *Record) error { r.Replicas = []string{"uk"}; return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rt, _ := reg.Get("acme.go.tito.io"); rt == nil { |
| + t.Fatal("home vanished") |
| + } else if rt.Replicas != "uk" { |
| + t.Fatalf("row replicas = %q after the record's set shrank, want %q — a stale placement keeps a retired region serving the account", rt.Replicas, "uk") |
| + } |
| + |
| + // And to none: the mirror deleted its last replica. |
| + if _, err := updateRecord(ctx, s, sqid, "tito", "acme.go.tito.io", func(r *Record) error { r.Replicas = nil; return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rt, _ := reg.Get("acme.go.tito.io"); rt == nil { |
| + t.Fatal("home vanished") |
| + } else if rt.Replicas != "" { |
| + t.Fatalf("row replicas = %q after the record cleared its set, want empty", rt.Replicas) |
| + } |
| +} |
| + |
| +// TestReplicaMarkMintsAndConverges: the row's replica mark is Record.Role's |
| +// shadow, and the mark decides how the edges file publishes the row — so it |
| +// converges from the record in BOTH directions. A re-mint carrying no role |
| +// must DEMOTE a row that carried one: a stale mark keeps steering treating |
| +// the row's box as a replica of an account that no longer has one there. |
| +func TestReplicaMarkMintsAndConverges(t *testing.T) { |
| + s, sqid := setup(t) |
| + replica := Record{Schema: 1, Host: "acme-uk.go.tito.io", Slug: "acme-uk", App: "tito", |
| + Account: sqid, Pool: "tito", Region: "uk", Channel: "stable", |
| + Role: "replica", CreatedAt: time.Now()} |
| + if err := CreateRecord(ctx, s, replica); err != nil { |
| + t.Fatal(err) |
| + } |
| + reg := regWith(t) |
| + if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if rt, _ := reg.Get("acme-uk.go.tito.io"); rt == nil { |
| + t.Fatal("replica not minted") |
| + } else if !rt.Replica { |
| + t.Fatal("minted row carries no replica mark") |
| + } |
| + |
| + // The role leaves the record; the mark follows. |
| + if _, err := updateRecord(ctx, s, sqid, "tito", "acme-uk.go.tito.io", func(r *Record) error { r.Role = ""; return nil }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "uk", Pools: []string{"tito"}, DataDir: "/data"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + rt, err := reg.Get("acme-uk.go.tito.io") |
| + if err != nil || rt == nil { |
| + t.Fatalf("get: %v", err) |
| + } |
| + if rt.Replica { |
| + t.Fatal("row keeps a replica mark its record no longer carries — the edges file would go on telling the fleet this box replicas an account it homes") |
| + } |
| +} |
| + |
| +// TestReplicaNeverEverywhereClass pins the residence rule's second door |
| +// (Fable review, finding 3): a replica is a process on ONE box — the one its |
| +// placement names — and everywhereClass's widening is the mint gate for |
| +// every non-fleet box. The class check refuses a replica FIRST (before any |
| +// kind or addr test), and everywhereRoute agrees, because the two must |
| +// never disagree about a row whose record is gone. |
| +func TestReplicaNeverEverywhereClass(t *testing.T) { |
| + // Static would pass the kind arm on its own — the role check must be |
| + // first and absolute. |
| + if everywhereClass(Record{Role: "replica", Kind: "static"}) { |
| + t.Error("everywhereClass(replica static) = true — a replica must never widen past its placement box") |
| + } |
| + if everywhereClass(Record{Role: "replica", Kind: "instance"}) { |
| + t.Error("everywhereClass(replica instance) = true") |
| + } |
| + // A public-addr replica would pass the dial test on its own; the shape |
| + // door refuses it, and the class check must not depend on that door |
| + // holding. |
| + if err := ValidateShape(Record{Role: "replica", Addr: "203.0.113.5:8090"}); err == nil { |
| + t.Error("ValidateShape(replica with a public addr) accepted — the everywhere-class second door") |
| + } |
| + if everywhereRoute(registry.Route{Replica: true, Kind: "static"}) { |
| + t.Error("everywhereRoute(replica row) = true — the removal-phase half of the same question") |
| + } |
| +} |
diff --git a/internal/provision/replica.go b/internal/provision/replica.go| new file mode 100644 |
| index 000000000..e2283bf86 |
| --- /dev/null |
| +++ b/internal/provision/replica.go |
| @@ -0,0 +1,105 @@ |
| +package provision |
| + |
| +// The replica provisioning surface (#162, edge checkout). The instance |
| +// record family's own verb, exactly as the discussion's Ask 2 frames it: a |
| +// replica is the sandbox verb with a different role and a chosen placement — |
| +// same app, same account, role=replica, placed on the box that carries the |
| +// placement code. |
| +// |
| +// The record the console's create door writes (apiinstances.go) and the |
| +// record a home's own code would write share ONE constructor, Replica, so |
| +// "what a replica record looks like" has exactly one answer: everything |
| +// copied from the home record's policy half (app, account, pool, channel), |
| +// Region = the replica's placement, Role = "replica", and ValidateShape run |
| +// over the result — the same rails the door applies, because neither side |
| +// trusts the other about what is in the bucket. |
| +// |
| +// The routing half of #162 lives in the reconciler and the edges file; the |
| +// lease/drain protocol that runs BETWEEN home and replica is titogo's own. |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// ErrReplicaOfReplica and ErrReplicaIncompleteHome are Replica's refusals: |
| +// a replica of a replica is not a shape the platform carries, and a home |
| +// link to an incomplete record would mint a replica whose account or app is |
| +// nothing the home ever was. |
| +var ( |
| + ErrReplicaOfReplica = errors.New("provision: record is itself a replica — a replica of a replica is not a shape the platform carries") |
| + ErrReplicaIncompleteHome = errors.New("provision: home record is not a complete instance record (host/app/account)") |
| +) |
| + |
| +// Replica turns a home-shaped record into the replica record |
| +// CreateRecordReplica writes, validating the result as it goes. host is the |
| +// replica's own host (one label + the app's instance domain — the same shape |
| +// a home claims), placement is the replica's placement code (ValidRegion's |
| +// vocabulary, the same one the create door validates as armed), home is the |
| +// home record as read from the bucket. The returned record has passed |
| +// ValidateShape — the proof ReplicaRecord's type carries. |
| +func Replica(home Record, host, placement string) (ReplicaRecord, error) { |
| + if home.Role == "replica" { |
| + return ReplicaRecord{}, ErrReplicaOfReplica |
| + } |
| + if home.Host == "" || home.App == "" || home.Account == "" { |
| + return ReplicaRecord{}, ErrReplicaIncompleteHome |
| + } |
| + rec := Record{ |
| + Schema: 1, |
| + Host: host, |
| + Slug: host, |
| + App: home.App, |
| + Account: home.Account, |
| + Pool: home.Pool, |
| + Region: placement, |
| + Channel: home.Channel, |
| + CreatedAt: time.Now().UTC(), |
| + Role: "replica", |
| + } |
| + if err := ValidateShape(rec); err != nil { |
| + return ReplicaRecord{}, err |
| + } |
| + return ReplicaRecord{rec}, nil |
| +} |
| + |
| +// ReplicaRecord is a Record that has passed the replica rails: ValidateShape |
| +// (which refuses a replica role on anything but a socket-backed instance) |
| +// has seen this exact value. Constructors return it; nothing else may build |
| +// one, so a writer that wants to stamp the role onto a record must go |
| +// through the same validation the door applies. |
| +type ReplicaRecord struct { |
| + Record |
| +} |
| + |
| +// CreateRecordReplica writes the replica record — create-only, the |
| +// CreateRecord contract (a lost race is release.ErrCASConflict, never a |
| +// silent overwrite). The record MUST have come from Replica: the type is |
| +// the proof ValidateShape's replica rails have already run, and this writer |
| +// re-checks nothing. |
| +func CreateRecordReplica(ctx context.Context, s release.Store, rec ReplicaRecord) error { |
| + return CreateRecord(ctx, s, rec.Record) |
| +} |
| + |
| +// ListRecordReplicas returns an app's records that ARE replicas — the |
| +// instance-API list the home instance walks to find whom it leases to, |
| +// addressable by Host + Region. Records are listed through ListRecords (the |
| +// same capped, path-validated fold the instance list uses) and filtered to |
| +// role=replica, so a home instance can never mistake a sibling home for its |
| +// own replica even before it checks the role on the wire view. |
| +func ListRecordReplicas(ctx context.Context, s release.Store, acct, app string, max int) ([]Record, error) { |
| + recs, _, err := ListRecords(ctx, s, acct, app, max) |
| + if err != nil { |
| + return nil, err |
| + } |
| + var out []Record |
| + for _, rec := range recs { |
| + if rec.Role == "replica" { |
| + out = append(out, rec) |
| + } |
| + } |
| + return out, nil |
| +} |
diff --git a/internal/registry/registry.go b/internal/registry/registry.go| index 2d1f935aa..e4d558d66 100644 |
| --- a/internal/registry/registry.go |
| +++ b/internal/registry/registry.go |
| @@ -261,6 +261,40 @@ type Route struct { |
| // different questions (SnapshotFingerprint's own doc comment). |
| Steering string |
| + // Replicas is the account's replica placements, comma-joined placement |
| + // codes in canonical (sorted, deduplicated) order — the registry half of |
| + // #162. "" on every row that predates the column = no replicas = |
| + // today's routing, byte for byte. Written by SetReplicas (the provision |
| + // pass's converge — the value is the record's, and this column must |
| + // track it, including down to "") and carried by routeFor at the mint. |
| + // |
| + // Read by the edges-file pass (a replica placement is where the account |
| + // is SERVED) and the steering converger (a replica placement earns a |
| + // latency record); never by the edge's own dispatch. A replica host's |
| + // own row is a separate route whose Region names its placement, exactly |
| + // as a home's row does — this column lives on the HOME's row, naming |
| + // where its replicas sit. |
| + // |
| + // Deliberately NOT in Fingerprint(), in Steering's exact shadow: it |
| + // changes what DNS answers and what the edges file publishes, not what |
| + // this box's routing map dials. SnapshotFingerprint covers it |
| + // automatically via routeCols. |
| + Replicas string |
| + |
| + // Replica marks the row as itself a replica — an edge-checkout shard of |
| + // the account (#162), holding a one-way mirror of it, selling against a |
| + // capacity lease from home. Written by routeFor at the mint from |
| + // Record.Role and converged by SetReplica; never set by an operator |
| + // verb. Read by the edges-file pass (the row's entries publish with the |
| + // replica mark, so steering's home-vs-replica fold can tell them apart) |
| + // and by the reconciler's removal-scope test — never by the edge's own |
| + // dispatch, which serves the row exactly as any other and never |
| + // hairpins it. |
| + // |
| + // Deliberately NOT in Fingerprint(), Steering's shadow again: a mark |
| + // changes what the edges file publishes, not what this box dials. |
| + Replica bool |
| + |
| CreatedAt int64 |
| } |
| @@ -471,6 +505,8 @@ CREATE TABLE IF NOT EXISTS routes ( |
| `ALTER TABLE routes ADD COLUMN args TEXT NOT NULL DEFAULT ''`, |
| `ALTER TABLE routes ADD COLUMN idle_seconds INTEGER NOT NULL DEFAULT 0`, |
| `ALTER TABLE routes ADD COLUMN max_run_seconds INTEGER NOT NULL DEFAULT 0`, |
| + `ALTER TABLE routes ADD COLUMN replicas TEXT NOT NULL DEFAULT ''`, |
| + `ALTER TABLE routes ADD COLUMN replica INTEGER NOT NULL DEFAULT 0`, |
| } { |
| if _, err := r.sql.Exec(stmt); err != nil && !strings.Contains(err.Error(), "duplicate column") { |
| return err |
| @@ -714,7 +750,7 @@ func (r *Registry) Accounts() ([]Account, error) { |
| } |
| // routeCols is the column list every route read and the one route write |
| -// share — 40 names, in this exact order. scanRoute's destinations, Upsert's |
| +// share — 42 names, in this exact order. scanRoute's destinations, Upsert's |
| // placeholder row and Upsert's argument list all count from here: adding a |
| // column means adding it in FOUR places, and a miscount is a silent column |
| // shift rather than an error (TestUpsertRoundTripsAllColumns is what catches |
| @@ -724,23 +760,24 @@ version, prev_version, app, pool, region, channel, hibernate, canary, |
| replication, next_due, backing, status_named, platform, dns_status, |
| dns_checked_at, cert_expires_at, account, environment, caps, tunnel, |
| provisioned, seed, unit, fleet_box, addrs, health, steering, gate, created_at, |
| -args, idle_seconds, max_run_seconds` |
| +args, idle_seconds, max_run_seconds, replicas, replica` |
| func scanRoute(scan func(...any) error) (Route, error) { |
| var t Route |
| - var hib, can, sn, plat, prov int |
| + var hib, can, sn, plat, prov, rep int |
| err := scan(&t.Host, &t.Slug, &t.Kind, &t.Socket, &t.Addr, &t.Status, |
| &t.LastError, &t.DBPath, &t.Version, &t.PrevVersion, &t.App, &t.Pool, |
| &t.Region, &t.Channel, &hib, &can, &t.Replication, &t.NextDue, &t.Backing, |
| &sn, &plat, &t.DNSStatus, &t.DNSCheckedAt, &t.CertExpiresAt, &t.Account, |
| &t.Environment, &t.Caps, &t.Tunnel, &prov, &t.Seed, &t.Unit, &t.FleetBox, |
| &t.Addrs, &t.Health, &t.Steering, &t.Gate, &t.CreatedAt, |
| - &t.Args, &t.IdleSeconds, &t.MaxRunSeconds) |
| + &t.Args, &t.IdleSeconds, &t.MaxRunSeconds, &t.Replicas, &rep) |
| t.Hibernate = hib != 0 |
| t.Canary = can != 0 |
| t.StatusNamed = sn != 0 |
| t.Platform = plat != 0 |
| t.Provisioned = prov != 0 |
| + t.Replica = rep != 0 |
| return t, err |
| } |
| @@ -933,6 +970,14 @@ func (r *Registry) Upsert(t Route) error { |
| if !t.Hibernate { |
| t.Replication = "" |
| } |
| + // #162's replica invariant, its clean form: a replica SET on a row that |
| + // IS itself a replica is a column that lies — a replica of a replica is |
| + // not a shape any supported writer produces, and a replica row's |
| + // placement is its own Region. Dropped silently, like the replication |
| + // rung above. |
| + if t.Replica { |
| + t.Replicas = "" |
| + } |
| b2i := func(b bool) int { |
| if b { |
| return 1 |
| @@ -941,7 +986,7 @@ func (r *Registry) Upsert(t Route) error { |
| } |
| _, err := r.sql.Exec(` |
| INSERT INTO routes (`+routeCols+`) |
| -VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) |
| +VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) |
| ON CONFLICT(host) DO UPDATE SET |
| slug=excluded.slug, kind=excluded.kind, socket=excluded.socket, |
| addr=excluded.addr, status=excluded.status, last_error=excluded.last_error, |
| @@ -963,6 +1008,7 @@ ON CONFLICT(host) DO UPDATE SET |
| unit=CASE WHEN excluded.unit = '' THEN routes.unit ELSE excluded.unit END, |
| fleet_box=CASE WHEN excluded.fleet_box = '' THEN routes.fleet_box ELSE excluded.fleet_box END, |
| addrs=excluded.addrs, health=excluded.health, steering=excluded.steering, |
| + replicas=excluded.replicas, |
| gate=CASE WHEN excluded.gate = '' THEN routes.gate ELSE excluded.gate END, |
| args=CASE WHEN excluded.args = '' THEN routes.args ELSE excluded.args END, |
| idle_seconds=CASE WHEN excluded.idle_seconds = 0 THEN routes.idle_seconds ELSE excluded.idle_seconds END, |
| @@ -973,7 +1019,8 @@ ON CONFLICT(host) DO UPDATE SET |
| b2i(t.StatusNamed), b2i(t.Platform), t.DNSStatus, t.DNSCheckedAt, |
| t.CertExpiresAt, t.Account, t.Environment, t.Caps, t.Tunnel, |
| b2i(t.Provisioned), t.Seed, t.Unit, t.FleetBox, t.Addrs, t.Health, |
| - t.Steering, t.Gate, t.CreatedAt, t.Args, t.IdleSeconds, t.MaxRunSeconds) |
| + t.Steering, t.Gate, t.CreatedAt, t.Args, t.IdleSeconds, t.MaxRunSeconds, |
| + t.Replicas, b2i(t.Replica)) |
| return err |
| } |
| @@ -1313,6 +1360,63 @@ func (r *Registry) SetSteering(host, steering string) error { |
| return nil |
| } |
| +// SetReplicas repoints a route's replica placement set (#162) as a narrow |
| +// UPDATE: the provision pass converges this column every tick from the |
| +// record, and the record's value is authoritative in BOTH directions — |
| +// including down to "". That asymmetry with the CASE-preserved columns |
| +// above is the whole point: a set that shrank (a replica deleted in a |
| +// region the account left) must actually shrink on the row, or the edges |
| +// file goes on publishing a replica placement the account no longer has — |
| +// a residence lie, not a cosmetic one. Errors on an unmatched host, |
| +// SetChannel's contract: the reconciler must notice the row vanishing |
| +// underneath rather than silently do nothing. |
| +func (r *Registry) SetReplicas(host, replicas string) error { |
| + res, err := r.sql.Exec(`UPDATE routes SET replicas = ? WHERE host = ?`, replicas, host) |
| + if err != nil { |
| + return err |
| + } |
| + n, err := res.RowsAffected() |
| + if err != nil { |
| + return err |
| + } |
| + if n == 0 { |
| + return fmt.Errorf("no route for host %q", host) |
| + } |
| + return nil |
| +} |
| + |
| +// SetReplica stamps or clears a route's replica mark (#162) as a narrow |
| +// UPDATE — the provision pass converges it from Record.Role. The mark is |
| +// never an operator verb (a role flip is a data-plane event; delete and |
| +// re-create is the supported transition, Record.Role's doc), so unlike |
| +// SetChannel there is no human-typing-hostnames caller to keep the |
| +// error-on-unmatched contract for — but it is kept anyway, SetChannel's |
| +// contract verbatim, because a reconcile pass that has just read the row |
| +// must notice it vanishing underneath, not silently do nothing. |
| +func (r *Registry) SetReplica(host string, replica bool) error { |
| + res, err := r.sql.Exec(`UPDATE routes SET replica = ? WHERE host = ?`, b2iDB(replica), host) |
| + if err != nil { |
| + return err |
| + } |
| + n, err := res.RowsAffected() |
| + if err != nil { |
| + return err |
| + } |
| + if n == 0 { |
| + return fmt.Errorf("no route for host %q", host) |
| + } |
| + return nil |
| +} |
| + |
| +// b2iDB is Upsert's b2i for the narrow setters, which sit outside that |
| +// function's closure. |
| +func b2iDB(b bool) int { |
| + if b { |
| + return 1 |
| + } |
| + return 0 |
| +} |
| + |
| // SetGate stamps or clears a route's password gate — the narrow UPDATE the |
| // provision converge uses (never a full-row Upsert, which preserves a set |
| // gate by design and so could not clear one). Errors on an unmatched host, |
diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go| index 3bdb2fee1..43353b324 100644 |
| --- a/internal/registry/registry_test.go |
| +++ b/internal/registry/registry_test.go |
| @@ -77,6 +77,7 @@ func TestUpsertRoundTripsAllColumns(t *testing.T) { |
| Hibernate: true, Canary: true, Replication: "batched", |
| NextDue: 4200, Backing: BackingUnit, Unit: "console.service", |
| Steering: "latency", |
| + Replicas: "ie,uk,us", |
| Gate: "$2a$10$distinct", |
| CreatedAt: 1700000000, |
| } |
| @@ -93,6 +94,30 @@ func TestUpsertRoundTripsAllColumns(t *testing.T) { |
| if *got != want { |
| t.Fatalf("round trip mismatch:\n got %+v\n want %+v", *got, want) |
| } |
| + |
| + // The replica mark and the replica set are the #162 pair, and Upsert's |
| + // invariant blanks the SET on a row that IS a replica (a replica of a |
| + // replica is not a shape the platform carries) — so the mark needs its |
| + // own round trip, on a row whose set is empty. |
| + wantReplica := Route{ |
| + Host: "r.example.com", Slug: "r", Kind: "instance", |
| + Addr: "127.0.0.1:8081", Status: "ready", |
| + App: "kass", Pool: "shared", Region: "uk", Channel: "stable", |
| + Replica: true, CreatedAt: 1700000001, |
| + } |
| + if err := reg.Upsert(wantReplica); err != nil { |
| + t.Fatal(err) |
| + } |
| + gotReplica, err := reg.Get("r.example.com") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if gotReplica == nil { |
| + t.Fatal("Get returned nil for the replica row") |
| + } |
| + if *gotReplica != wantReplica { |
| + t.Fatalf("replica row round trip mismatch:\n got %+v\n want %+v", *gotReplica, wantReplica) |
| + } |
| } |
| // An addrs or health change must move the edge's Fingerprint — the edge only |
diff --git a/internal/steering/deployment_test.go b/internal/steering/deployment_test.go| index d1c82d933..8b478cabc 100644 |
| --- a/internal/steering/deployment_test.go |
| +++ b/internal/steering/deployment_test.go |
| @@ -20,7 +20,7 @@ func edgesBody(t *testing.T, box, eip, region string) []byte { |
| "eip": eip, |
| "region": region, |
| "generation": "2026-08-23T09:00:00Z", |
| - "hosts": map[string]any{"app.example.com": map[string]string{"role": "home"}}, |
| + "hosts": map[string]any{"app.example.com": map[string]any{"role": "home"}}, |
| }) |
| if err != nil { |
| t.Fatal(err) |
diff --git a/internal/steering/steering.go b/internal/steering/steering.go| index 2da352900..2e8dc7708 100644 |
| --- a/internal/steering/steering.go |
| +++ b/internal/steering/steering.go |
| @@ -55,13 +55,32 @@ type desiredRecord struct { |
| // fixes for this task), and homemap.go's reader does not yet carry the |
| // Region field this fold needs (it is a Task 1 addition homemap.go has no |
| // consumer for). Hosts is deliberately absent: steering is any-edge- |
| -// any-app (the spec's home-boxed-kinds-steer-via-backhaul rule), so this |
| -// fold never needs to know which hosts an edge homes. |
| +// any-app (the spec's home-boxed-kinds-steer-via-backhaul rule). That |
| +// was true until #162: the replica-aware skip in desiredState DOES need to |
| +// know which hosts an edge serves, because the replica mark is exactly what |
| +// removes a record — a box whose file names a steered host with |
| +// replica:true is serving it from a read-only checkout shard, not as a |
| +// home. |
| + |
| +// edgeHostView is one entry of an edges file's hosts map, as the steering |
| +// fold reads it. A named type rather than an anonymous struct so tests and |
| +// readers spell it once; the only field this fold consults is the replica |
| +// mark — role values stay the writer's and homemap's business. |
| +type edgeHostView struct { |
| + Replica bool `json:"replica,omitempty"` |
| +} |
| + |
| type edgesFileView struct { |
| V int `json:"v"` |
| Box string `json:"box"` |
| EIP string `json:"eip"` |
| Region string `json:"region,omitempty"` |
| + // Hosts is the per-host map the file publishes (the writer's |
| + // internal/host/edgeHost shape; see edgeHostView for what survives the |
| + // decode). The mark rides the ENTRY (edgeHost.Replica on the writer |
| + // side, #162); a file-level flag would have no meaning — a box can |
| + // serve one account as a home and another as a replica. |
| + Hosts map[string]edgeHostView `json:"hosts"` |
| } |
| // parseEdgesFileView decodes one box's edges file body into the fields |
| @@ -355,7 +374,19 @@ func desiredState(routes []registry.Route, edges []edgesFileView, appsDomain str |
| for _, r := range steeredHosts(routes, domain) { |
| name := fqdn(r.Host) |
| + // Replica-aware skip (#162): steering is any-edge-any-app (a far edge |
| + // terminates and BACKHAULS to the home — that is the whole design), |
| + // so every armed edge earns a record for a steered host exactly as it |
| + // did before #162, and the ONLY entries removed are the replica mark's: |
| + // a box whose file names the host WITH replica:true is serving it from |
| + // a read-only checkout shard, and steering buyers' whole sessions there |
| + // would be wrong in a way backhaul is not. Entries with no replica |
| + // mark — every pre-#162 file, every home — keep their records |
| + // byte-identically. |
| for _, e := range armed { |
| + if h, named := e.Hosts[r.Host]; named && h.Replica { |
| + continue |
| + } |
| out = append(out, desiredRecord{Name: name, EIP: e.EIP, Box: e.Box, Region: e.Region}) |
| } |
| } |
diff --git a/internal/steering/steering_test.go b/internal/steering/steering_test.go| index a1a1e4aeb..71f19edd9 100644 |
| --- a/internal/steering/steering_test.go |
| +++ b/internal/steering/steering_test.go |
| @@ -25,16 +25,73 @@ func steeredRoute(host string) registry.Route { |
| return registry.Route{Host: host, Steering: "latency"} |
| } |
| +// TestDesiredStateReplicasDoNotSteer pins the #162 half of the fold: an |
| +// account whose replica box publishes the host marked replica gets records |
| +// from EVERY armed edge EXCEPT that one — the pre-#162 rule is any-edge- |
| +// any-app (far edges backhaul), so the ONLY entries a replica mark removes |
| +// are its own. The synthetic edge.<apps-domain> records keep ALL armed |
| +// edges regardless (every edge is a cache front). |
| +func TestDesiredStateReplicasDoNotSteer(t *testing.T) { |
| + dublin := edgeHomes("dublin", "1.2.3.4", "eu-west-1", "app.example.com") |
| + oregon := edge("oregon", "5.6.7.8", "us-west-2") |
| + oregon.Hosts = map[string]edgeHostView{ |
| + "app.example.com": {Replica: true}, |
| + } |
| + |
| + got := desiredState([]registry.Route{steeredRoute("app.example.com")}, |
| + []edgesFileView{dublin, oregon}, "example.com") |
| + |
| + want := []desiredRecord{ |
| + {Name: "edge.example.com.", EIP: "1.2.3.4", Box: "dublin", Region: "eu-west-1"}, |
| + {Name: "edge.example.com.", EIP: "5.6.7.8", Box: "oregon", Region: "us-west-2"}, |
| + {Name: "app.example.com.", EIP: "1.2.3.4", Box: "dublin", Region: "eu-west-1"}, |
| + } |
| + if !reflect.DeepEqual(got, want) { |
| + t.Fatalf("desiredState() =\n%#v\nwant\n%#v", got, want) |
| + } |
| + |
| + // A host named by NO box still earns records from every armed edge: |
| + // steering is any-edge-any-app, and the far edge backhauls to the home |
| + // whether or not this box holds a row for the host. |
| + got = desiredState([]registry.Route{steeredRoute("never-published.example.com")}, |
| + []edgesFileView{dublin, oregon}, "example.com") |
| + want = []desiredRecord{ |
| + {Name: "edge.example.com.", EIP: "1.2.3.4", Box: "dublin", Region: "eu-west-1"}, |
| + {Name: "edge.example.com.", EIP: "5.6.7.8", Box: "oregon", Region: "us-west-2"}, |
| + {Name: "never-published.example.com.", EIP: "1.2.3.4", Box: "dublin", Region: "eu-west-1"}, |
| + {Name: "never-published.example.com.", EIP: "5.6.7.8", Box: "oregon", Region: "us-west-2"}, |
| + } |
| + if !reflect.DeepEqual(got, want) { |
| + t.Fatalf("unpublished-host desiredState() =\n%#v\nwant\n%#v", got, want) |
| + } |
| +} |
| + |
| func edge(box, eip, region string) edgesFileView { |
| return edgesFileView{V: 1, Box: box, EIP: eip, Region: region} |
| } |
| +// edgeHomes is edge() with the homed-host set filled in: the tests that |
| +// drive desiredState's per-host fold need Hosts, which pre-#162 was |
| +// deliberately absent (the fold never asked who served what) and which the |
| +// replica-aware fold now reads. Hosts uses the writer's own key shape; |
| +// every entry is home-marked (Replica clear) unless a test says otherwise. |
| +func edgeHomes(box, eip, region string, homes ...string) edgesFileView { |
| + e := edge(box, eip, region) |
| + e.Hosts = map[string]edgeHostView{} |
| + for _, h := range homes { |
| + e.Hosts[h] = edgeHostView{} |
| + } |
| + return e |
| +} |
| + |
| // TestDesiredState is the brief's table: one steered host x two armed |
| // edges = two records; unsteered host absent; edge.<apps-domain> present |
| // whenever >=1 edge is armed, absent at zero. |
| func TestDesiredState(t *testing.T) { |
| - dublin := edge("dublin", "1.2.3.4", "eu-west-1") |
| - oregon := edge("oregon", "5.6.7.8", "us-west-2") |
| + dublin := edgeHomes("dublin", "1.2.3.4", "eu-west-1", |
| + "app.example.com", "steered.example.com", "example.com") |
| + oregon := edgeHomes("oregon", "5.6.7.8", "us-west-2", |
| + "app.example.com", "steered.example.com") |
| tests := []struct { |
| name string |
| @@ -310,7 +367,7 @@ func TestDesiredStateBlankAppsDomainExcludesEveryPerHostRecord(t *testing.T) { |
| // identical Box/EIP on both sides and the log would just repeat the one |
| // folded Name. |
| func TestDesiredStateCaseFoldDedupe(t *testing.T) { |
| - dublin := edge("dublin", "1.2.3.4", "eu-west-1") |
| + dublin := edgeHomes("dublin", "1.2.3.4", "eu-west-1", "app.example.com") |
| buf := captureLog(t) |
| routes := []registry.Route{ |
| @@ -405,7 +462,12 @@ func TestParseEdgesFileView(t *testing.T) { |
| if err != nil { |
| t.Fatalf("parseEdgesFileView(%q) unexpected error: %v", tt.body, err) |
| } |
| - if got != tt.want { |
| + // Field-by-field, not struct equality: Hosts is a map, and a |
| + // comparable-struct check died the day #162 added it. The |
| + // scalar fields are the ones this test pins; Hosts is read |
| + // only for the replica mark in desiredState, pinned there. |
| + if got.V != tt.want.V || got.Box != tt.want.Box || got.EIP != tt.want.EIP || |
| + got.Region != tt.want.Region { |
| t.Fatalf("parseEdgesFileView(%q) = %+v, want %+v", tt.body, got, tt.want) |
| } |
| }) |
diff --git a/mutation_message_test.go b/mutation_message_test.go| index 9198da66b..d8b5cba0f 100644 |
| --- a/mutation_message_test.go |
| +++ b/mutation_message_test.go |
| @@ -132,7 +132,7 @@ func TestMutationMessagesTravelOnJSONWrites(t *testing.T) { |
| if _, err := s.configApply(ctx, a, "titogo", apiConfigApply{Set: map[string]appconfig.Var{"MODE": {Kind: "env", Value: "live"}}, Message: "switch to live mode"}); err != nil { |
| t.Fatal(err) |
| } |
| - if _, err := s.instanceCreate(ctx, a, "titogo", "web.example.test", "", "edge", "", "", nil, nil, "add the web copy"); err != nil { |
| + if _, err := s.instanceCreate(ctx, a, "titogo", "web.example.test", "", "edge", "", "", nil, nil, nil, "add the web copy"); err != nil { |
| t.Fatal(err) |
| } |