amadan
console: a delayed create whose replay already committed answers success, not pending
Corrects Task 3's create door. When the original request's SQLite commit is refused, allocRecordLocked now asks committedElsewhere: if recovery committed the allocation and the judged record under the key is valid at the allocation's incarnation, the create is a known success and the door answers 201 with that record. Anything else (a tombstone, a body under a retired slot, a row not committed) stays pending as before, and the log line no longer says "pending; the record stands" for the committed case. TestRecoveryReplaysCreateAndTheLateWriteIsHarmless now requires 201. The write inventory re-pins allocRecordLocked and the internal/console envelope is refreshed in place (PROPOSED).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 files changed,
+56
−8
internal/console/lifetimes_create.go+25 −1internal/console/lifetimes_create_test.go+19 −5internal/provision/self_serve_moves1_census_test.go+10 −1internal/provision/testdata/plan14-root-reads-self-serve-moves-1.tsv+2 −1
diff --git a/internal/console/lifetimes_create.go b/internal/console/lifetimes_create.go| index fe87aee6c..be68cd66b 100644 |
| --- a/internal/console/lifetimes_create.go |
| +++ b/internal/console/lifetimes_create.go |
| @@ -324,12 +324,36 @@ func (s *Server) allocRecordLocked(ctx context.Context, a Allocation) (*provisio |
| } |
| rec.RecordSeq = seq |
| if err := s.store.SetAllocationState(a.OpID, lifetimeCommitted); err != nil { |
| - log.Printf("console: lifetimes: allocation %s of %s: commit: %v (pending; the record stands)", a.OpID, a.Host, err) |
| + if landed, ok := s.committedElsewhere(ctx, a); ok { |
| + // Recovery (a second console) committed this allocation first: |
| + // the create is done, and its record is still this lifetime's. |
| + log.Printf("console: lifetimes: allocation %s of %s: committed already by recovery; the record stands", a.OpID, a.Host) |
| + return landed, nil |
| + } |
| + log.Printf("console: lifetimes: allocation %s of %s: commit: %v (pending)", a.OpID, a.Host, err) |
| return nil, errLifetimePending |
| } |
| return &rec, nil |
| } |
| +// committedElsewhere is the answer to a refused commit: the allocation's row |
| +// is committed (recovery got there first) and the judged record under its |
| +// key is valid at its incarnation, so the create is a known success and the |
| +// record is returned. Anything else (a row still pending, a terminal state |
| +// other than committed, a record its slot no longer vouches for, a read that |
| +// fails) is not known and stays pending. |
| +func (s *Server) committedElsewhere(ctx context.Context, a Allocation) (*provision.Record, bool) { |
| + rows, err := s.store.allocations(`op_id = ?`, a.OpID) |
| + if err != nil || len(rows) != 1 || rows[0].State != lifetimeCommitted { |
| + return nil, false |
| + } |
| + landed, _, err := s.readRecord(ctx, a.Account, a.App, a.Host) |
| + if err != nil || landed.Incarnation != a.Incarnation || landed.Retired { |
| + return nil, false |
| + } |
| + return landed, true |
| +} |
| + |
| // ---- the slot-bound orphan retirement --------------------------------------- |
| // slotDisposition is what a slot retirement found. |
diff --git a/internal/console/lifetimes_create_test.go b/internal/console/lifetimes_create_test.go| index 5f5e37fdf..142552c62 100644 |
| --- a/internal/console/lifetimes_create_test.go |
| +++ b/internal/console/lifetimes_create_test.go |
| @@ -1049,18 +1049,32 @@ func TestRecoveryReplaysCreateAndTheLateWriteIsHarmless(t *testing.T) { |
| l := newLtOn(t) |
| gate, hook := pauseNth(t, "put", recKey(ltApp, ltOne), 1) |
| l.bucket.setHook(hook) |
| - answered := make(chan int, 1) |
| - go func() { code, _ := l.create(t, ltOne); answered <- code }() |
| + type answer struct { |
| + code int |
| + body string |
| + } |
| + answered := make(chan answer, 1) |
| + go func() { code, body := l.create(t, ltOne); answered <- answer{code, body} }() |
| waitFor(t, gate.entered, "the first console's record write") |
| // The first console is gone; the second recovers from SQLite. |
| recoverOn(t, l.restart(t)) |
| a := oneAlloc(t, l.fx.st, ltOne, lifetimeCommitted) |
| _, etag := rawOf(t, l.fx.rel, ltApp, ltOne) |
| _, slotETag := slotOf(t, l.fx.rel, ltOne) |
| - // Now the first console's create-only write arrives. |
| + // Now the first console's create-only write arrives. It loses to the |
| + // replay, which recovery committed: the create is a known success, and |
| + // the door answers it as a first-time create does. |
| gate.open() |
| - if code := <-answered; code != http.StatusBadGateway && code != http.StatusCreated { |
| - t.Fatalf("the late request answered %d", code) |
| + late := <-answered |
| + if late.code != http.StatusCreated { |
| + t.Fatalf("the late request answered %d %s, want 201", late.code, late.body) |
| + } |
| + var view struct { |
| + Host string `json:"host"` |
| + Incarnation uint64 `json:"incarnation"` |
| + } |
| + if err := json.Unmarshal([]byte(late.body), &view); err != nil || view.Host != ltOne || view.Incarnation != 1 { |
| + t.Fatalf("the late 201 body = %s (%v), want the record of %s at 1", late.body, err, ltOne) |
| } |
| requireServing(t, l.fx.rel, ltApp, ltOne, 1, a.OpID) |
| if _, now := rawOf(t, l.fx.rel, ltApp, ltOne); now != etag { |
diff --git a/internal/provision/self_serve_moves1_census_test.go b/internal/provision/self_serve_moves1_census_test.go| index ee0625e53..2f124ffae 100644 |
| --- a/internal/provision/self_serve_moves1_census_test.go |
| +++ b/internal/provision/self_serve_moves1_census_test.go |
| @@ -216,6 +216,15 @@ package provision |
| // - reconcile.go:mintHeld: key-site as before (the claim's record key); |
| // the held mint carries the hold's lifetime pair in its INSERT; no |
| // bucket write. |
| +// |
| +// Task 9 fix round 1 (a delayed create whose replay already committed), |
| +// reviewed disposition (PROPOSED): no write site added, moved or made raw. |
| +// - lifetimes_create.go:Server.allocRecordLocked: record-writer through |
| +// WriteRecord as before, re-pinned in place. When its SQLite commit is |
| +// refused it asks committedElsewhere, which reads the allocation row and |
| +// the record through readRecord (judged) and writes nothing; a row |
| +// committed by recovery and a valid record at the allocation's |
| +// incarnation answer the create as a success. |
| func plan14SelfServeMoves1WriteCensusInventory() []censusEntry { |
| removed := []struct{ file, unit string }{ |
| {"internal/provision/provision.go", "GetRecordETag"}, |
| @@ -282,7 +291,7 @@ func plan14SelfServeMoves1WriteCensusInventory() []censusEntry { |
| {"internal/console/apiinstances.go", "Server.handleAPIInstanceCreate", "record-writer", "key:CreateRecord", "7a840049171d50f3575eaff17b45503905c8617d9ffdec88cc4a1b9aebcdbfd9"}, |
| {"internal/console/domainattach.go", "Server.attachDomainAs", "record-writer", "key:CreateRecord", "7c44eac1ca3c50d91c90616699910f27b7a8ea2d5417fbe4f624b5d12dc0ddb3"}, |
| {"internal/console/apiinstanceseed.go", "Server.createSeedRecord", "record-writer", "key:CreateRecord", "a6fabf99b13e7e2302dbf2c30fb7c782f76f787141a5cc12580fde59805dfd4a"}, |
| - {"internal/console/lifetimes_create.go", "Server.allocRecordLocked", "record-writer", "key:RecordKey,key:WriteRecord", "d7e42956bd01014463e00b26395e7eda97a31187080f1daa9effab20fea9282a"}, |
| + {"internal/console/lifetimes_create.go", "Server.allocRecordLocked", "record-writer", "key:RecordKey,key:WriteRecord", "004b8ea62416916a9655132797604fbe0a277ff935b99df9ca7ea3791f916305"}, |
| {"internal/console/lifetimes_create.go", "Server.fenceRecord", "record-writer", "key:RecordKey,key:WriteRecord", "efee3393b8d4a362fc395271671f59c9c75d6f7dc2f6970494725dbecb3275d8"}, |
| {"internal/console/lifetimes_create.go", "Server.allocCleanLocked", "key-site", "key:RecordKey", "035cf56d8f69517b66dc8ea6f9121d463e482d7dd05daf2b0c494aef1255f522"}, |
| {"internal/console/lifetimes_create.go", "Server.deleteUnderRetiredSlot", "key-site", "key:RecordKey", "16d160495d03f8ea606f2d4bb724dedcde6677cc4d5428ce76770a429c300109"}, |
diff --git a/internal/provision/testdata/plan14-root-reads-self-serve-moves-1.tsv b/internal/provision/testdata/plan14-root-reads-self-serve-moves-1.tsv| index a5fb423bf..b47b2eb43 100644 |
| --- a/internal/provision/testdata/plan14-root-reads-self-serve-moves-1.tsv |
| +++ b/internal/provision/testdata/plan14-root-reads-self-serve-moves-1.tsv |
| @@ -45,7 +45,7 @@ internal/console/store_gate.go claimSQLCensus legacy accepted-legacy accepted-le |
| @remove internal/console/tempstore.go Store.CreateTemporaryAppInRegion |
| @removed internal/console/tempstore.go Store.CreateTemporaryAppInRegion H3 census-ruling-PROPOSED-self-serve-moves-1;rev3=PROPOSED-self-serve-moves-1;basis=constructor remains declared but has no direct read; app insertion and lifetime minting moved to insertAppRow, classified above; proposed removal only |
| # Task 2 envelopes, rev3=PROPOSED-self-serve-moves-1: internal/console covers the lifetime store, app constructors, lifetime projection and table-scoped gate labels; internal/provision (above) adds only ErrLifetimeExhausted beyond Task 1. Registry fields are unchanged. |
| -@coverage internal/console 85a457565795e720527b4c0eb78eacea224b02d8db088a24041e758f50b50f5a |
| +@coverage internal/console 33115e588cea10142e02514e086040a7d1d4505e654dab760bbff38e885c3e61 |
| # Task 6a: rev3=PROPOSED-self-serve-moves-1; reviewed provision envelope now includes bounded record-then-slot observations, judged read/list/update gates and signed-counter exhaustion at slot read. No removal-loop, host or console change; registry fields and the legacy seal stand. |
| internal/provision/rootrecord.go IndexRecords authority B3,C rev3=PROPOSED-self-serve-moves-1;basis=retain every discovered candidate owner key per host for fresh lifetime judgement at Find; partial discovery still retries and no verdict or body is cached e44ac431865bf38baa4105f66c9cb23014cc3c7ffcd41d5385244cc76fc4c0b9 |
| internal/provision/rootrecord.go RecordIndex.Find authority B3,C rev3=PROPOSED-self-serve-moves-1;basis=observe each candidate through the context judge, refuse corruption and read failures, require one valid owner and validate its schema and shape; nil-judge duplicates retain permission refusal 760eed59cbd6a8ddab1098ce3a5f7a4272ecda81c17cc079dc135aba3a85a8f8 |
| @@ -192,3 +192,4 @@ internal/registry/remote_dispatch.go Registry.dispatchRemote hint E2 rev3=PROPOS |
| internal/host/edgesfile.go edgesFileWriteClaims authority F,J1 rev3=PROPOSED-self-serve-moves-1;basis=root-route;kept=F;scope=protected-decision-only; the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): region-ha-2a's body, which with the box's lifetimes switch on also publishes each served row's lifetime pair (registry:Incarnation and registry:ServingSeq, two new sites on the rows it already lists) and an alias's or served catch-all's primary's pair, and names that primary as target_host whatever the switch; a report the edge orders unstamped entries by, not a decision; no record, /data, release or claim read added; F,J1 authority retained; self-serve moves 1 Task 8 registry:Account@11578,registry:Alias@5526,registry:Alias@5561,registry:Alias@5983,registry:Alias@6140,registry:Alias@6210,registry:Alias@6242,registry:Alias@6915,registry:Alias@9270,registry:App@11590,registry:DNSStatus@5658,registry:DNSStatus@6149,registry:Host@11008,registry:Host@3187,registry:Host@5618,registry:Host@6463,registry:Host@6899,registry:Host@7072,registry:Host@9317,registry:Host@9364,registry:Host@9414,registry:Host@9449,registry:Host@9498,registry:Incarnation@3147,registry:Platform@2926,registry:Provisioned@2940,registry:Replica@2964,registry:ServingSeq@3162,registry:Status@893,whole:Route@339,whole:Route@3441,whole:Route@367,whole:Route@3734,whole:RouteAlias@5511,whole:RouteAlias@9223,whole:RoutePath@3749,whole:WildcardDomain@9403 be1fe7b02a3f7977c7b210f72ee445bb2fa3458498416efa8186cf9ce79752a4 |
| @legacy 5ca6674843515dd4b3587ac65b3d8d380f9aa1bfa0a7d5b50b02f01ad18e90c2 internal/host/edgesfile.go edgesFileLoop legacy accepted-legacy accepted-legacy;rev3=PROPOSED-self-serve-moves-1;basis=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): census-ruling-moves-offline's body, which also takes the box's lifetimes switch (runAgent's boxJudge != nil) and hands it to every edgesFileWriteClaims pass; read site unchanged, offset moved; a report, not a decision; self-serve moves 1 Task 8 file:Read@2757 65c1d42c39f7f69035ccc6c2a6ce92f155f26d7c7af4b82b4633e215557615e7 |
| internal/host/edgesfile.go edgesFileLoop legacy accepted-legacy accepted-legacy;rev3=PROPOSED-self-serve-moves-1;basis=the agent (/opt/carlos/carlos agent, carlos-edge.service, user carlos): census-ruling-moves-offline's body, which also takes the box's lifetimes switch (runAgent's boxJudge != nil) and hands it to every edgesFileWriteClaims pass; read site unchanged, offset moved; a report, not a decision; self-serve moves 1 Task 8 file:Read@2757 65c1d42c39f7f69035ccc6c2a6ce92f155f26d7c7af4b82b4633e215557615e7 |
| +# Task 9 fix round 1 (a delayed create whose replay already committed answers success), rev3=PROPOSED-self-serve-moves-1: allocRecordLocked asks committedElsewhere when its SQLite commit is refused (one SQLite read of its own allocation row, one judged record read through readRecord; no write). No root read, class, legacy row or seal changes; the internal/console envelope is refreshed in place above. |