amadan
plan: claim Plan 8b rev 4, answers review round 3 (three P2)
Co-Authored-By: Codex <noreply@openai.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 file changed,
+1121
−100
docs/superpowers/plans/2026-10-06-host-lifecycle-claim-8b-floor-roster.md+1121 −100
CI passed — run details
diff --git a/docs/superpowers/plans/2026-10-06-host-lifecycle-claim-8b-floor-roster.md b/docs/superpowers/plans/2026-10-06-host-lifecycle-claim-8b-floor-roster.md| index 7bba1a402..fb44ac457 100644 |
| --- a/docs/superpowers/plans/2026-10-06-host-lifecycle-claim-8b-floor-roster.md |
| +++ b/docs/superpowers/plans/2026-10-06-host-lifecycle-claim-8b-floor-roster.md |
| @@ -1,6 +1,6 @@ |
| # Host lifecycle claim, Plan 8b: the floor, the roster and live fencing |
| -Rev 3 answers review round 2 (four P2). |
| +Rev 4 answers review round 3 (three P2). |
| Implementation plan, 2026-10-06. Branch to implement: `feat/hostclaim-floor`, |
| from `feat/hostclaim-leases` at |
| @@ -170,6 +170,109 @@ Add the two requested SystemRelease fields, the separate console artifact mappin |
| **Step 1: add the failing tests.** |
| +### Create `internal/release/systempublication_test.go` |
| + |
| +```go |
| +package release |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "errors" |
| + "os" |
| + "testing" |
| +) |
| + |
| +type heldSystemPublicationStore struct { |
| + Store |
| + key string |
| + entered chan struct{} |
| + resume chan struct{} |
| +} |
| + |
| +func (s heldSystemPublicationStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { |
| + if key == s.key { |
| + close(s.entered) |
| + <-s.resume |
| + } |
| + return s.Store.Put(ctx, key, raw, etag) |
| +} |
| + |
| +func TestSystemPublicationCancellationWinsDelayedCAS(t *testing.T) { |
| + for _, existing := range []bool{false, true} { |
| + name := "absent" |
| + if existing { |
| + name = "existing" |
| + } |
| + t.Run(name, func(t *testing.T) { |
| + ctx := t.Context() |
| + key, pub := testKeys(t) |
| + sg := RootSigner(key) |
| + store := NewDirStore(t.TempDir()) |
| + if existing { |
| + if _, err := PromoteSystem(ctx, store, sg, "router", "stable", testSystemRelease("v1.0.0"), SystemPromoteOptions{}); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + prepared, err := PrepareSystemPromotion(ctx, store, sg, "router", "stable", testSystemRelease("v1.1.0"), SystemPromoteOptions{}, "cancelled-update") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + held := heldSystemPublicationStore{Store: store, key: SystemPointerKey("router", "stable"), entered: make(chan struct{}), resume: make(chan struct{})} |
| + done := make(chan error, 1) |
| + go func() { _, err := PublishSystemPublication(ctx, held, pub, prepared); done <- err }() |
| + <-held.entered |
| + if err := CancelSystemPublication(ctx, store, sg, prepared); err != nil { |
| + close(held.resume) |
| + t.Fatal(err) |
| + } |
| + close(held.resume) |
| + if err := <-done; !errors.Is(err, ErrCASConflict) { |
| + t.Fatalf("delayed CAS: %v", err) |
| + } |
| + if err := CancelSystemPublication(ctx, store, sg, prepared); err != nil { |
| + t.Fatalf("cancel retry: %v", err) |
| + } |
| + p, etag, err := GetSystemPointer(ctx, store, pub, "router", "stable") |
| + if existing { |
| + if err != nil || p.Current.Version != "v1.0.0" { |
| + t.Fatal(p, err) |
| + } |
| + } else if !errors.Is(err, os.ErrNotExist) || etag == "" { |
| + t.Fatalf("tombstone read: %s %v", etag, err) |
| + } |
| + if _, err := PromoteSystem(ctx, store, sg, "router", "stable", testSystemRelease("v1.2.0"), SystemPromoteOptions{}); err != nil { |
| + t.Fatalf("new promotion after fence: %v", err) |
| + } |
| + if _, err := PublishSystemPublication(ctx, store, pub, prepared); !errors.Is(err, ErrCASConflict) { |
| + t.Fatalf("cancelled activation resurrected: %v", err) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestSystemPublicationCancellationCannotUndoCommittedUpdate(t *testing.T) { |
| + ctx := t.Context() |
| + key, pub := testKeys(t) |
| + sg := RootSigner(key) |
| + store := NewDirStore(t.TempDir()) |
| + p, err := PrepareSystemPromotion(ctx, store, sg, "router", "stable", testSystemRelease("v1.0.0"), SystemPromoteOptions{}, "committed-update") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := PublishSystemPublication(ctx, store, pub, p); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := CancelSystemPublication(ctx, store, sg, p); !errors.Is(err, ErrCASConflict) { |
| + t.Fatal(err) |
| + } |
| + raw, _, err := store.Get(ctx, SystemPointerKey("router", "stable")) |
| + if err != nil || !bytes.Equal(raw, p.After) { |
| + t.Fatal("committed publication changed", err) |
| + } |
| +} |
| +``` |
| + |
| ### Create `internal/hostfloor/floor_test.go` |
| ```go |
| @@ -268,6 +371,7 @@ Run this now and expect failure before the implementation is present: |
| ```sh |
| go test ./internal/hostfloor -run 'TestFloorPair|TestPendingAndLost|TestCanonicalRelease' -count=1 |
| +go test ./internal/release -run '^TestSystemPublication' -count=1 |
| ``` |
| **Step 2: add the implementation.** |
| @@ -388,6 +492,104 @@ func CreateExact(ctx context.Context, s Store, key string, raw []byte) error { |
| } |
| ``` |
| +### Create `internal/release/systempublication.go` |
| + |
| +```go |
| +package release |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "crypto/ed25519" |
| + "errors" |
| + "os" |
| +) |
| + |
| +// SystemPublication retains the exact CAS and signed bytes across retries. |
| +type SystemPublication struct { |
| + Component string `json:"component"` |
| + Channel string `json:"channel"` |
| + ID string `json:"id"` |
| + ExpectedETag string `json:"expected_etag"` |
| + Before []byte `json:"before,omitempty"` |
| + After []byte `json:"after"` |
| +} |
| + |
| +func (p SystemPublication) Verify(pub ed25519.PublicKey) (*SystemPointer, error) { |
| + next, err := verifySystemPointer(pub, p.After, p.Component, p.Channel) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if next.ActivationID != p.ID || next.CancellationID != "" { |
| + return nil, ErrBadSystemPointer |
| + } |
| + return next, nil |
| +} |
| + |
| +func PublishSystemPublication(ctx context.Context, s Store, pub ed25519.PublicKey, p SystemPublication) (*SystemPointer, error) { |
| + next, err := p.Verify(pub) |
| + if err != nil { |
| + return nil, err |
| + } |
| + key := SystemPointerKey(p.Component, p.Channel) |
| + _, err = s.Put(ctx, key, p.After, p.ExpectedETag) |
| + if err == nil { |
| + return next, nil |
| + } |
| + // A failed response can follow a committed write. Only these exact signed |
| + // bytes prove completion; a different pointer never authorizes another CAS. |
| + raw, _, readErr := s.Get(ctx, key) |
| + if readErr == nil && bytes.Equal(raw, p.After) { |
| + return next, nil |
| + } |
| + return nil, err |
| +} |
| + |
| +// CancelSystemPublication competes with publication on its original pointer |
| +// ETag. The fence must be durable before the actor reservation is removed. |
| +func CancelSystemPublication(ctx context.Context, s Store, sg Signer, p SystemPublication) error { |
| + if p.ID == "" { |
| + return ErrBadSystemPointer |
| + } |
| + if _, err := p.Verify(sg.verifyPub()); err != nil { |
| + return err |
| + } |
| + before := &SystemPointer{Schema: 1, Component: p.Component, Channel: p.Channel} |
| + if len(p.Before) != 0 { |
| + var err error |
| + before, err = verifySystemPointer(sg.verifyPub(), p.Before, p.Component, p.Channel) |
| + if err != nil { |
| + return err |
| + } |
| + } |
| + before.CancellationID = p.ID |
| + fence, err := SignRootEnvelope(sg, before) |
| + if err != nil { |
| + return err |
| + } |
| + key := SystemPointerKey(p.Component, p.Channel) |
| + raw, etag, err := s.Get(ctx, key) |
| + if err == nil && bytes.Equal(raw, fence) { |
| + return nil |
| + } |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + return err |
| + } |
| + if etag != p.ExpectedETag || (err == nil && bytes.Equal(raw, p.After)) { |
| + return ErrCASConflict |
| + } |
| + _, err = s.Put(ctx, key, fence, p.ExpectedETag) |
| + if err == nil { |
| + return nil |
| + } |
| + raw, _, readErr := s.Get(ctx, key) |
| + if readErr == nil && bytes.Equal(raw, fence) { |
| + return nil |
| + } |
| + return err |
| +} |
| +``` |
| + |
| ### Create `internal/hostfloor/disk.go` |
| ```go |
| @@ -987,7 +1189,26 @@ func ReadEnrollment(path string) (Enrollment, error) { |
| ```diff |
| --- a/internal/release/systempointer.go |
| +++ b/internal/release/systempointer.go |
| -@@ -36,4 +36,7 @@ |
| +@@ -22,8 +22,10 @@ |
| +- Schema int `json:"schema"` |
| +- Component string `json:"component"` |
| +- Channel string `json:"channel"` |
| +- Current SystemRelease `json:"current"` |
| +- History []SystemRelease `json:"history,omitempty"` |
| +- Spread string `json:"spread"` |
| +- PromotedAt time.Time `json:"promoted_at"` // of THIS write; the box's replay guard reads only this |
| +- Audit []Promotion `json:"audit,omitempty"` |
| ++ ActivationID string `json:"activation_id,omitempty"` |
| ++ CancellationID string `json:"cancellation_id,omitempty"` |
| ++ Schema int `json:"schema"` |
| ++ Component string `json:"component"` |
| ++ Channel string `json:"channel"` |
| ++ Current SystemRelease `json:"current"` |
| ++ History []SystemRelease `json:"history,omitempty"` |
| ++ Spread string `json:"spread"` |
| ++ PromotedAt time.Time `json:"promoted_at"` // of THIS write; the box's replay guard reads only this |
| ++ Audit []Promotion `json:"audit,omitempty"` |
| +@@ -36,4 +38,7 @@ |
| - Version string `json:"version"` |
| - Artifacts map[string]SystemArtifact `json:"artifacts"` // keyed by "<goos>-<goarch>" |
| - Source string `json:"source,omitempty"` |
| @@ -999,25 +1220,98 @@ func ReadEnrollment(path string) (Enrollment, error) { |
| + Artifacts map[string]SystemArtifact `json:"artifacts"` // keyed by "<goos>-<goarch>" |
| + Source string `json:"source,omitempty"` |
| + ReleasedAt time.Time `json:"released_at"` // audit only; never compared |
| -@@ -102,0 +106,5 @@ |
| -+ } |
| +@@ -103,0 +109,5 @@ |
| + for platform, artifact := range r.ConsoleArtifacts { |
| + if !systemPlatformRe.MatchString(platform) || artifact.Key != "bootstrap/console/"+r.Version+"/carlos-"+platform || !systemSHA256Re.MatchString(artifact.SHA256) { |
| + return ErrBadSystemPointer |
| + } |
| -@@ -287 +295 @@ |
| ++ } |
| +@@ -129,0 +140,3 @@ |
| ++ } |
| ++ if p.CancellationID != "" && p.Current.Version == "" { |
| ++ return nil |
| +@@ -228,0 +242,3 @@ |
| ++ if p.CancellationID != "" && p.Current.Version == "" { |
| ++ return nil, etag, os.ErrNotExist |
| ++ } |
| +@@ -242,0 +259,8 @@ |
| ++ prepared, err := PrepareSystemPromotion(ctx, s, sg, component, channel, rel, o, "") |
| ++ if err != nil { |
| ++ return nil, err |
| ++ } |
| ++ return PublishSystemPublication(ctx, s, sg.verifyPub(), prepared) |
| ++} |
| ++ |
| ++func PrepareSystemPromotion(ctx context.Context, s Store, sg Signer, component, channel string, rel SystemRelease, o SystemPromoteOptions, id string) (SystemPublication, error) { |
| +@@ -249 +273 @@ |
| +- return nil, err |
| ++ return SystemPublication{}, err |
| +@@ -254,3 +278,6 @@ |
| +- return nil, err |
| +- } |
| +- next := &SystemPointer{Component: component, Channel: channel, Current: rel, Spread: o.Spread.String()} |
| ++ return SystemPublication{}, err |
| ++ } |
| ++ if old != nil && old.CancellationID != "" && old.Current.Version == "" { |
| ++ old = nil |
| ++ } |
| ++ next := &SystemPointer{ActivationID: id, Component: component, Channel: channel, Current: rel, Spread: o.Spread.String()} |
| +@@ -271,6 +298,3 @@ |
| +- return nil, err |
| +- } |
| +- if _, err := s.Put(ctx, key, wrapper, etag); err != nil { |
| +- return nil, err |
| +- } |
| +- return next, nil |
| ++ return SystemPublication{}, err |
| ++ } |
| ++ return SystemPublication{Component: component, Channel: channel, ID: id, ExpectedETag: etag, Before: body, After: wrapper}, nil |
| +@@ -287 +311,9 @@ |
| -func RollbackSystem(ctx context.Context, s Store, sg Signer, component, channel, actor string) (*SystemPointer, error) { |
| +func RollbackSystem(ctx context.Context, s Store, sg Signer, component, channel, actor string, expectedETag ...string) (*SystemPointer, error) { |
| -@@ -291,0 +300,3 @@ |
| ++ prepared, err := PrepareSystemRollback(ctx, s, sg, component, channel, actor, "", expectedETag...) |
| ++ if err != nil { |
| ++ return nil, err |
| ++ } |
| ++ return PublishSystemPublication(ctx, s, sg.verifyPub(), prepared) |
| ++} |
| ++ |
| ++func PrepareSystemRollback(ctx context.Context, s Store, sg Signer, component, channel, actor, id string, expectedETag ...string) (SystemPublication, error) { |
| +@@ -291 +323,4 @@ |
| +- return nil, err |
| ++ return SystemPublication{}, err |
| + } |
| + if len(expectedETag) > 1 || (len(expectedETag) == 1 && etag != expectedETag[0]) { |
| -+ return nil, ErrCASConflict |
| ++ return SystemPublication{}, ErrCASConflict |
| +@@ -295 +330 @@ |
| +- return nil, err |
| ++ return SystemPublication{}, err |
| +@@ -298,3 +333,3 @@ |
| +- return nil, ErrNoSystemHistory |
| +- } |
| +- next := &SystemPointer{ |
| ++ return SystemPublication{}, ErrNoSystemHistory |
| ++ } |
| ++ next := &SystemPointer{ActivationID: id, |
| +@@ -307,7 +342,4 @@ |
| +- return nil, err |
| +- } |
| +- if _, err := s.Put(ctx, key, wrapper, etag); err != nil { |
| +- return nil, err |
| +- } |
| +- return next, nil |
| +-} |
| ++ return SystemPublication{}, err |
| ++ } |
| ++ return SystemPublication{Component: component, Channel: channel, ID: id, ExpectedETag: etag, Before: body, After: wrapper}, nil |
| ++} |
| ``` |
| **Step 4: verify the task.** |
| ```sh |
| go test ./internal/hostfloor -run 'TestFloorPair|TestPendingAndLost|TestCanonicalRelease' -count=1 |
| +go test ./internal/release -run '^TestSystemPublication' -count=1 |
| ``` |
| ## Task 2: The signed roster, verified enrollment, census and reservations |
| @@ -1763,6 +2057,62 @@ go test ./internal/hostactors -count=1 |
| **Step 2: add the implementation.** |
| +### Create `internal/hostactors/activation_publication.go` |
| + |
| +```go |
| +package hostactors |
| + |
| +import ( |
| + "context" |
| + "reflect" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +func (d Directory) PublishActivation(ctx context.Context, id string) (*release.SystemPointer, error) { |
| + o, err := d.Read(ctx) |
| + if err != nil { |
| + return nil, err |
| + } |
| + h, ok := o.Roster.Activations[id] |
| + if !ok || h.Publication == nil || h.Publication.ID != id { |
| + return nil, ErrRoster |
| + } |
| + return release.PublishSystemPublication(ctx, d.Store, d.PublicKey, *h.Publication) |
| +} |
| + |
| +func (d Directory) CancelActivation(ctx context.Context, sg release.Signer, id string) error { |
| + o, err := d.Read(ctx) |
| + if err != nil { |
| + return err |
| + } |
| + h, ok := o.Roster.Activations[id] |
| + if !ok { |
| + return nil |
| + } |
| + if h.Publication == nil || h.Publication.ID != id { |
| + return ErrRoster |
| + } |
| + if err := release.CancelSystemPublication(ctx, d.Store, sg, *h.Publication); err != nil { |
| + return err |
| + } |
| + fresh, err := d.Read(ctx) |
| + if err != nil { |
| + return err |
| + } |
| + if _, ok := fresh.Roster.Activations[id]; !ok { |
| + return nil |
| + } |
| + if !reflect.DeepEqual(h, fresh.Roster.Activations[id]) { |
| + return ErrRoster |
| + } |
| + next := Clone(fresh.Roster) |
| + delete(next.Activations, id) |
| + _, err = d.Replace(ctx, sg, fresh, next) |
| + return err |
| +} |
| +``` |
| + |
| ### Create `internal/hostactors/activation_complete.go` |
| ```go |
| @@ -2092,6 +2442,9 @@ func (d Directory) Reserve(ctx context.Context, sg release.Signer, o Observed, h |
| } |
| } |
| for _, other := range r.Activations { |
| + if h.Publication != nil && other.Publication != nil && h.Publication.Component == other.Publication.Component && h.Publication.Channel == other.Publication.Channel { |
| + return Observed{}, ErrRoster |
| + } |
| if overlaps(h, other) { |
| return Observed{}, ErrRoster |
| } |
| @@ -2102,6 +2455,9 @@ func (d Directory) Reserve(ctx context.Context, sg release.Signer, o Observed, h |
| return Observed{}, ErrRoster |
| } |
| for _, other := range r.Activations { |
| + if h.Publication != nil && other.Publication != nil && h.Publication.Component == other.Publication.Component && h.Publication.Channel == other.Publication.Channel { |
| + return Observed{}, ErrRoster |
| + } |
| if overlaps(h, other) { |
| return Observed{}, ErrRoster |
| } |
| @@ -3133,14 +3489,17 @@ type Console struct { |
| DeploymentBucket string `json:"deployment_bucket"` |
| } |
| type Reservation struct { |
| - ID string `json:"id"` |
| - Host string `json:"host"` |
| - OpID string `json:"op_id"` |
| - ExpectedClaimETag string `json:"expected_claim_etag"` |
| - FloorSHA256 string `json:"floor_sha256"` |
| - Actors map[string]string `json:"actors"` |
| - Releases map[string]Report `json:"releases"` |
| - Hashes map[string][]string `json:"hashes,omitempty"` |
| + Publication *release.SystemPublication `json:"publication,omitempty"` |
| + RequestSHA256 string `json:"request_sha256,omitempty"` |
| + UpdateMode string `json:"update_mode,omitempty"` |
| + ID string `json:"id"` |
| + Host string `json:"host"` |
| + OpID string `json:"op_id"` |
| + ExpectedClaimETag string `json:"expected_claim_etag"` |
| + FloorSHA256 string `json:"floor_sha256"` |
| + Actors map[string]string `json:"actors"` |
| + Releases map[string]Report `json:"releases"` |
| + Hashes map[string][]string `json:"hashes,omitempty"` |
| } |
| type Roster struct { |
| EnrollmentAttempts map[string]EnrollmentAttempt `json:"enrollment_attempts,omitempty"` |
| @@ -3210,6 +3569,9 @@ func (r Roster) Validate(bucket string) error { |
| if !segment(id) || h.ID != id || len(h.FloorSHA256) != 64 || len(h.Actors) == 0 || len(h.Releases) != len(h.Actors) { |
| return ErrRoster |
| } |
| + if h.Publication != nil && (h.Publication.ID != id || len(h.RequestSHA256) != 64 || (h.UpdateMode != "promote" && h.UpdateMode != "rollback")) { |
| + return ErrRoster |
| + } |
| for key, enrollment := range h.Actors { |
| expected := "" |
| if ref, ok := strings.CutPrefix(key, "box:"); ok { |
| @@ -3826,6 +4188,71 @@ func TestGuardRecoveryRefusesControllersAfterBegin(t *testing.T) { |
| } |
| } |
| } |
| + |
| +func TestGuardRejectsOldPointerForUnpublishedActivation(t *testing.T) { |
| + f, id := guardRecoveryFixture(t) |
| + a := f.a |
| + ctx := t.Context() |
| + sg := release.RootSigner(f.priv) |
| + old, _, err := release.GetSystemPointer(ctx, f.store, f.pub, "router", a.channel) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + tx, err := a.guardTransaction() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var roster hostactors.Roster |
| + if err := release.VerifyRootEnvelope(f.pub, tx.Roster, &roster); err != nil { |
| + t.Fatal(err) |
| + } |
| + publication, err := release.PrepareSystemPromotion(ctx, f.store, sg, "router", a.channel, old.Current, release.SystemPromoteOptions{}, id) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + h := roster.Activations[id] |
| + h.Publication = &publication |
| + h.RequestSHA256 = strings.Repeat("d", 64) |
| + h.UpdateMode = "promote" |
| + roster.Activations[id] = h |
| + raw, err := release.SignRootEnvelope(sg, roster) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := f.store.Put(ctx, hostactors.Key, raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + identity, _, err := f.store.Get(ctx, release.SystemPointerKey("router", a.channel)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PublishSystemIdentity(ctx, f.store, f.pub, identity); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.MkdirAll(filepath.Join(a.etcDir, "carlos"), 0755); err != nil { |
| + t.Fatal(err) |
| + } |
| + enrollment, _ := json.Marshal(hostfloor.Enrollment{Box: "box", Enrollment: tx.Enrollment, InstanceID: "instance"}) |
| + if err := hostfloor.Durable(filepath.Join(a.etcDir, "carlos", "host-actor.json"), enrollment, 0644); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "activation.json")); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := a.prepareGuard(ctx, old); err == nil { |
| + t.Fatal("old same-target pointer consumed unpublished activation") |
| + } |
| + if _, err := os.Stat(filepath.Join(a.floor.Dir, "activation.json")); !os.IsNotExist(err) { |
| + t.Fatalf("refusal wrote transaction: %v", err) |
| + } |
| + current, err := release.PublishSystemPublication(ctx, f.store, f.pub, publication) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := a.prepareGuard(ctx, current); err != nil { |
| + t.Fatal("published activation refused", err) |
| + } |
| +} |
| ``` |
| ### Create `internal/hostexec/updateidentity_test.go` |
| @@ -4323,6 +4750,9 @@ func (a *systemApplier) prepareGuard(ctx context.Context, target *release.System |
| actor := "box:" + enrollment.Box |
| id := "" |
| for name, h := range roster.Activations { |
| + if h.Publication != nil && target.ActivationID != name { |
| + continue |
| + } |
| hashes := h.Hashes[actor] |
| if h.Actors[actor] == enrollment.Enrollment && len(hashes) == 3 && hashes[1] == target.Current.Artifacts[a.platform].SHA256 { |
| if id != "" { |
| @@ -5380,7 +5810,7 @@ go test ./internal/hostfence -count=1 |
| ## Task 5: Operator resolution, readmission and console/update wiring |
| -Replace both placeholder HTTP handlers and the CLI's impossible-success branch. Inspection returns a signed challenge with readable consequences. The operator observes the enrolled instance after inspection, explicitly confirms the attestation and submits the same envelope. --yes never manufactures an attestation about compute or disk destruction. The console has no EC2 client. Accepted losses are durable before their claim CAS; a claim-write failure leaves global quarantine in place. Readmission validates the fresh private report and exact retirement receipt, all retained declarations and complete claim inventory before clearing the global deny. Report publishing runs at startup and five-minute intervals outside NewServer. System promote and rollback reserve exact affected actors before changing a pointer; rollback also binds the pointer ETag read before reservation. |
| +Replace both placeholder HTTP handlers and the CLI's impossible-success branch. Inspection returns a signed challenge with readable consequences. The operator observes the enrolled instance after inspection, explicitly confirms the attestation and submits the same envelope. --yes never manufactures an attestation about compute or disk destruction. The console has no EC2 client. Accepted losses are durable before their claim CAS; a claim-write failure leaves global quarantine in place. Readmission validates the fresh private report and exact retirement receipt, all retained declarations and complete claim inventory before clearing the global deny. Report publishing runs at startup and five-minute intervals outside NewServer. System promote and rollback persist a resumable activation and reserve exact affected actors before changing a pointer. Both operations retain the original pointer ETag and exact signed publication bytes. |
| Persist root-signed acceptance at |
| `control/host-loss-acceptances/<declaration-id>.json` before the roster CAS |
| @@ -5391,56 +5821,285 @@ materializes the loss record after quarantine, and preserves those times. |
| An unrelated roster change uses a fresh CAS with the same box identity and |
| prior quarantine revision. A competing declaration for that box defeats the |
| prepared revision and requires new inspection; it adds no orphan loss ID. |
| -Readmission still requires every quarantined declaration to be committed. |
| +When a prepared loss has no retained claim reference and its expected claim |
| +ETag has changed, persist `conflicted` without changing its accepted evidence |
| +or applying stale consequences. A retained `loss_refs` entry proves an earlier |
| +successful CAS and completes committed bookkeeping. A conflicted retry remains |
| +a conflict. Readmission accepts committed or conflicted declarations, retains |
| +their records and roster loss IDs, and requires exact retirement evidence for |
| +conflicted declarations. Before any readmission mutation, verify the enrolled |
| +box's termination fence authority. A valid fence, malformed evidence or a read |
| +failure refuses readmission. |
| + |
| +Each update reservation stores its request digest, promotion or rollback mode, |
| +original pointer ETag and exact signed publication bytes. A retry with the same |
| +request resumes that activation even after a console restart. A different |
| +request for the channel refuses while it is reserved. After a failed PUT, |
| +reading back exactly the prepared bytes proves publication; unavailable or |
| +different bytes retain the reservation and return an error. Rollback retries |
| +resume the stored publication before reading history, so a lost response cannot |
| +pop another release. |
| + |
| +`CancelActivation` first wins a CAS against that original pointer position, |
| +then removes the reservation by roster CAS. The signed cancellation retains |
| +the prior release and changes the pointer bytes. For an originally absent |
| +pointer it writes a retained cancellation tombstone; upgraded readers report |
| +absence while retaining its ETag, and older readers refuse the empty release. |
| +Never delete the tombstone or retry publication against a newer ETag. A |
| +published activation cannot be cancelled and remains reserved until verified |
| +completion. A crash after cancellation's pointer CAS can retry the same fence |
| +and finish removing the reservation. Failure to establish that fence leaves |
| +the reservation in place. |
| + |
| +For publication-backed reservations, root requires the pointer's exact |
| +`activation_id` before preparing the update. An older pointer for the same |
| +target hash cannot consume an unpublished reservation. Existing reservations |
| +without publication metadata retain their earlier matching rule. |
| **Step 1: add the failing tests.** |
| -### Create `internal/hostresolve/service_test.go` |
| +### Create `internal/console/hostactivation_test.go` |
| ```go |
| -package hostresolve |
| +package console |
| import ( |
| + "bytes" |
| "context" |
| "crypto/ed25519" |
| - "crypto/rand" |
| "encoding/json" |
| "errors" |
| - "fmt" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| "github.com/carlosframework/platform/internal/hostactors" |
| - "github.com/carlosframework/platform/internal/hostclaim" |
| - "github.com/carlosframework/platform/internal/hostfence" |
| "github.com/carlosframework/platform/internal/hostfloor" |
| - "github.com/carlosframework/platform/internal/hostquarantine" |
| "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/status" |
| - "strings" |
| - "testing" |
| - "time" |
| ) |
| -type failClaimStore struct { |
| +type activationFaultStore struct { |
| release.Store |
| - fail bool |
| - crash string |
| + fail bool |
| + commit bool |
| + hide bool |
| } |
| -func (s *failClaimStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { |
| - if s.fail && strings.HasPrefix(key, "control/hosts/") { |
| - return "", errors.New("claim write unavailable") |
| +func (s *activationFaultStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { |
| + if key != release.SystemPointerKey("router", "stable") || !s.fail { |
| + return s.Store.Put(ctx, key, raw, etag) |
| } |
| - if s.crash == "acceptance" && strings.HasPrefix(key, "control/host-loss-acceptances/") || s.crash == "before-quarantine" && key == hostactors.Key || s.crash == "loss-create" && strings.HasPrefix(key, "control/host-loss/") { |
| - return "", errors.New("crash before durable write") |
| + s.fail = false |
| + if s.commit { |
| + if _, err := s.Store.Put(ctx, key, raw, etag); err != nil { |
| + return "", err |
| + } |
| } |
| - next, err := s.Store.Put(ctx, key, raw, etag) |
| - if err == nil && s.crash == "after-quarantine" && key == hostactors.Key { |
| - return "", errors.New("crash after durable quarantine") |
| + return "", errors.New("publication response lost") |
| +} |
| +func (s *activationFaultStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if s.hide && key == release.SystemPointerKey("router", "stable") && !s.fail { |
| + s.hide = false |
| + return nil, "", errors.New("publication read unavailable") |
| } |
| - return next, err |
| + return s.Store.Get(ctx, key) |
| } |
| -func resolveFixture(t *testing.T) (Service, *failClaimStore, *time.Time, string) { |
| + |
| +func activationFixture(t *testing.T) (*Server, hostactors.Directory, release.SystemRelease) { |
| t.Helper() |
| - ctx := context.Background() |
| + ctx := t.Context() |
| + pub, key, err := ed25519.GenerateKey(nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + store := release.NewDirStore(t.TempDir()) |
| + s := &Server{cfg: Config{Releases: store, Key: key, Bucket: "bucket"}} |
| + sg := release.RootSigner(key) |
| + old := release.SystemRelease{Version: "v1.0.0", Source: "release:v1.0.0", ReleaseTag: "v1.0.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("v1.0.0", "linux-amd64"), SHA256: strings.Repeat("a", 64)}}} |
| + next := release.SystemRelease{Version: "v1.1.0", Source: "release:v1.1.0", ReleaseTag: "v1.1.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("v1.1.0", "linux-amd64"), SHA256: strings.Repeat("b", 64)}}} |
| + for _, rel := range []release.SystemRelease{next, old} { |
| + if _, err := release.PromoteSystem(ctx, store, sg, "router", "stable", rel, release.SystemPromoteOptions{}); err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, _, err := store.Get(ctx, release.SystemPointerKey("router", "stable")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PublishSystemIdentity(ctx, store, pub, raw); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if err := hostfloor.Publish(ctx, store, sg, "bucket", hostfloor.Floor{Min: 1, MinRelease: "v1.0.0"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + floor, err := hostfloor.Read(ctx, store, pub, "bucket") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + enrollment := strings.Repeat("e", 32) |
| + roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Publication: "published", FloorSHA256: floor.Digest, Boxes: map[string]hostactors.Box{"box": {BoxID: "box", Namespace: "platform", Label: "box", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-box"}, Enrollment: enrollment, EvidenceSHA256: strings.Repeat("f", 64), State: "active"}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| + raw, err := release.SignRootEnvelope(sg, roster) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := store.Put(ctx, hostactors.Key, raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + box := status.Box{GeneratedAt: time.Now(), Metrics: &status.BoxMetrics{Enrollment: enrollment, InstanceID: "i-box", Release: old.Version, ReleaseTag: old.ReleaseTag, Platform: "linux-amd64", BinarySHA256: old.Artifacts["linux-amd64"].SHA256, MovesProtocol: 1, BootID: "boot", FloorSHA256: floor.Digest, FloorGuardReady: true, HostClaimEnabled: true, SystemChannel: "stable"}} |
| + raw, err = json.Marshal(box) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := store.Put(ctx, "status/boxes/box.json", raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + return s, hostactors.Directory{Store: store, PublicKey: pub, Bucket: "bucket"}, next |
| +} |
| + |
| +func TestHostUpdateReservationRetriesPublication(t *testing.T) { |
| + for _, fault := range []string{"crash", "failed", "committed", "unreadable"} { |
| + t.Run(fault, func(t *testing.T) { |
| + s, d, target := activationFixture(t) |
| + id, err := s.reserveHostUpdate(t.Context(), &target, "stable", "promote", release.SystemPromoteOptions{}) |
| + if err != nil || id == "" { |
| + t.Fatal(id, err) |
| + } |
| + first, err := d.Read(t.Context()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if fault != "crash" { |
| + broken := &activationFaultStore{Store: d.Store, fail: true, commit: fault != "failed", hide: fault == "unreadable"} |
| + s.cfg.Releases = broken |
| + _, err = s.promoteHostUpdate(t.Context(), target, "router", "stable", release.SystemPromoteOptions{}) |
| + if (err == nil) != (fault == "committed") { |
| + t.Fatalf("first publication: %v", err) |
| + } |
| + } |
| + // A new console process has no in-memory job or reservation state. |
| + s = &Server{cfg: Config{Releases: d.Store, Key: s.cfg.Key, Bucket: "bucket"}} |
| + p, err := s.promoteHostUpdate(t.Context(), target, "router", "stable", release.SystemPromoteOptions{}) |
| + if err != nil || p.ActivationID != id || p.Current.Version != target.Version { |
| + t.Fatal(p, err) |
| + } |
| + after, err := d.Read(t.Context()) |
| + if err != nil || len(after.Roster.Activations) != 1 { |
| + t.Fatal(after, err) |
| + } |
| + if !bytes.Equal(first.Roster.Activations[id].Publication.After, after.Roster.Activations[id].Publication.After) { |
| + t.Fatal("retry replaced publication identity") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestHostRollbackResumesWithoutPoppingAgain(t *testing.T) { |
| + s, d, _ := activationFixture(t) |
| + broken := &activationFaultStore{Store: d.Store, fail: true} |
| + s.cfg.Releases = broken |
| + if _, err := s.rollbackHostUpdate(t.Context(), "router", "stable", "operator"); err == nil { |
| + t.Fatal("missing injected failure") |
| + } |
| + before, err := d.Read(t.Context()) |
| + if err != nil || len(before.Roster.Activations) != 1 { |
| + t.Fatal(before, err) |
| + } |
| + var id string |
| + for id = range before.Roster.Activations { |
| + } |
| + s.cfg.Releases = d.Store |
| + first, err := s.rollbackHostUpdate(t.Context(), "router", "stable", "operator") |
| + if err != nil || first.ActivationID != id || first.Current.Version != "v1.1.0" { |
| + t.Fatal(first, err) |
| + } |
| + second, err := s.rollbackHostUpdate(t.Context(), "router", "stable", "operator") |
| + if err != nil || second.ActivationID != id || len(second.History) != len(first.History) { |
| + t.Fatal(second, err) |
| + } |
| +} |
| + |
| +func TestHostUpdateCancellationBlocksDelayedPublication(t *testing.T) { |
| + s, d, target := activationFixture(t) |
| + id, err := s.reserveHostUpdate(t.Context(), &target, "stable", "promote", release.SystemPromoteOptions{}) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + before, err := d.Read(t.Context()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stale := *before.Roster.Activations[id].Publication |
| + if err := d.CancelActivation(t.Context(), release.RootSigner(s.cfg.Key), id); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := release.PublishSystemPublication(t.Context(), d.Store, d.PublicKey, stale); !errors.Is(err, release.ErrCASConflict) { |
| + t.Fatalf("delayed publication: %v", err) |
| + } |
| + after, err := d.Read(t.Context()) |
| + if err != nil || len(after.Roster.Activations) != 0 { |
| + t.Fatal(after, err) |
| + } |
| + p, err := s.promoteHostUpdate(t.Context(), target, "router", "stable", release.SystemPromoteOptions{}) |
| + if err != nil || p.ActivationID == id { |
| + t.Fatal(p, err) |
| + } |
| + if err := d.CancelActivation(t.Context(), release.RootSigner(s.cfg.Key), p.ActivationID); !errors.Is(err, release.ErrCASConflict) { |
| + t.Fatalf("cancelled published activation: %v", err) |
| + } |
| + after, err = d.Read(t.Context()) |
| + if err != nil || len(after.Roster.Activations) != 1 { |
| + t.Fatal(after, err) |
| + } |
| +} |
| +``` |
| + |
| +### Create `internal/hostresolve/service_test.go` |
| + |
| +```go |
| +package hostresolve |
| + |
| +import ( |
| + "context" |
| + "crypto/ed25519" |
| + "crypto/rand" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostfence" |
| + "github.com/carlosframework/platform/internal/hostfloor" |
| + "github.com/carlosframework/platform/internal/hostquarantine" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/status" |
| + "strings" |
| + "testing" |
| + "time" |
| +) |
| + |
| +type failClaimStore struct { |
| + release.Store |
| + fail bool |
| + crash string |
| +} |
| + |
| +func (s *failClaimStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { |
| + if s.fail && strings.HasPrefix(key, "control/hosts/") { |
| + return "", errors.New("claim write unavailable") |
| + } |
| + if s.crash == "acceptance" && strings.HasPrefix(key, "control/host-loss-acceptances/") || s.crash == "before-quarantine" && key == hostactors.Key || s.crash == "loss-create" && strings.HasPrefix(key, "control/host-loss/") { |
| + return "", errors.New("crash before durable write") |
| + } |
| + next, err := s.Store.Put(ctx, key, raw, etag) |
| + if err == nil && s.crash == "after-quarantine" && key == hostactors.Key { |
| + return "", errors.New("crash after durable quarantine") |
| + } |
| + return next, err |
| +} |
| +func resolveFixture(t *testing.T) (Service, *failClaimStore, *time.Time, string) { |
| + t.Helper() |
| + ctx := context.Background() |
| pub, key, _ := ed25519.GenerateKey(rand.Reader) |
| store := &failClaimStore{Store: release.NewDirStore(t.TempDir())} |
| signer := release.RootSigner(key) |
| @@ -5652,10 +6311,259 @@ func prepareReadmission(t *testing.T, s Service, now time.Time, loss hostclaim.L |
| } |
| ``` |
| +### Create `internal/hostresolve/review3_test.go` |
| + |
| +```go |
| +package hostresolve |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "reflect" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostfloor" |
| + "github.com/carlosframework/platform/internal/hostquarantine" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/status" |
| +) |
| + |
| +func TestPreparedLossClaimProgressAllowsRetiredReadmission(t *testing.T) { |
| + for _, retired := range []bool{false, true} { |
| + t.Run(map[bool]string{false: "missing-retirement", true: "retired"}[retired], func(t *testing.T) { |
| + ctx := t.Context() |
| + s, store, now, host := resolveFixture(t) |
| + in := submitted(t, s, now, host) |
| + store.fail = true |
| + if _, err := s.Declare(ctx, "operator", host, in); err == nil { |
| + t.Fatal("claim failure not reached") |
| + } |
| + store.fail = false |
| + prepared, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) |
| + if err != nil || prepared.Record.State != "prepared" { |
| + t.Fatal(prepared, err) |
| + } |
| + before, err := hostclaim.Read(ctx, store, host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + // The retirement owner completes the exact local chore while quarantined. |
| + chore := before.Claim.Obligations[0] |
| + progressed, err := hostclaim.Update(ctx, store, before, hostclaim.CompleteObligation(chore.Op, chore.ID)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err = s.Declare(ctx, "operator", host, in); !errors.Is(err, hostclaim.ErrConflict) { |
| + t.Fatal("stale declaration must refuse", err) |
| + } |
| + conflicted, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) |
| + if err != nil || conflicted.Record.State != "conflicted" { |
| + t.Fatal("stale loss not settled", conflicted, err) |
| + } |
| + want := prepared.Record |
| + want.State = "conflicted" |
| + if !reflect.DeepEqual(want, conflicted.Record) { |
| + t.Fatal("accepted history changed") |
| + } |
| + if _, err = s.Declare(ctx, "operator", host, in); !errors.Is(err, hostclaim.ErrConflict) { |
| + t.Fatal("conflicted retry changed outcome", err) |
| + } |
| + unchanged, err := hostclaim.Read(ctx, store, host) |
| + if err != nil || unchanged.ETag != progressed.ETag { |
| + t.Fatal("stale consequences applied", err) |
| + } |
| + prepareReadmission(t, s, *now, conflicted.Record) |
| + if !retired { |
| + key, _ := hostquarantine.RetirementKey("dead", conflicted.Record.Enrollment, conflicted.Record.QuarantineRevision) |
| + raw, etag, err := store.Get(ctx, key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var receipt hostquarantine.RetirementCompletion |
| + if err = json.Unmarshal(raw, &receipt); err != nil { |
| + t.Fatal(err) |
| + } |
| + receipt.Entries = []hostquarantine.RetirementEntry{} |
| + if err := receipt.Validate(); err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, _ = json.Marshal(receipt) |
| + if _, err = store.Put(ctx, key, raw, etag); err != nil { |
| + t.Fatal(err) |
| + } |
| + digest := hostfloor.Digest(raw) |
| + raw, etag, err = store.Get(ctx, "status/boxes/dead.json") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var report status.Box |
| + if err = json.Unmarshal(raw, &report); err != nil { |
| + t.Fatal(err) |
| + } |
| + report.Metrics.RetirementReceiptSHA256 = digest |
| + raw, _ = json.Marshal(report) |
| + if _, err = store.Put(ctx, "status/boxes/dead.json", raw, etag); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + roster, err := s.Directory.Read(ctx) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + b := roster.Roster.Boxes["dead"] |
| + err = s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs) |
| + if (err == nil) != retired { |
| + t.Fatal("readmission must require retirement of conflicted loss", err) |
| + } |
| + after, err := s.Directory.Read(ctx) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + state := "quarantined" |
| + if retired { |
| + state = "active" |
| + } |
| + if after.Roster.Boxes["dead"].State != state || !reflect.DeepEqual(after.Roster.Boxes["dead"].LossIDs, b.LossIDs) { |
| + t.Fatal("state or loss history changed", after) |
| + } |
| + retained, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) |
| + if err != nil || !reflect.DeepEqual(retained.Record, conflicted.Record) { |
| + t.Fatal("readmission erased history", err) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +type fenceReadmitStore struct { |
| + release.Store |
| + unreadable bool |
| + writes int |
| +} |
| + |
| +func (s *fenceReadmitStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if s.unreadable && strings.HasPrefix(key, "control/box-fences/") { |
| + return nil, "", errors.New("fence unavailable") |
| + } |
| + return s.Store.Get(ctx, key) |
| +} |
| +func (s *fenceReadmitStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { |
| + s.writes++ |
| + return s.Store.Put(ctx, key, raw, etag) |
| +} |
| +func TestReadmitRefusesTerminationAuthorityBeforeMutation(t *testing.T) { |
| + for _, fault := range []string{"terminated", "malformed", "wrong-writer", "unreadable"} { |
| + t.Run(fault, func(t *testing.T) { |
| + ctx := t.Context() |
| + s, store, now, host := resolveFixture(t) |
| + in := submitted(t, s, now, host) |
| + if _, err := s.Declare(ctx, "operator", host, in); err != nil { |
| + t.Fatal(err) |
| + } |
| + loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + prepareReadmission(t, s, *now, loss.Record) |
| + roster, err := s.Directory.Read(ctx) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + b := roster.Roster.Boxes["dead"] |
| + fence := map[string]any{"v": 1, "box": "dead", "enrollment": b.Enrollment, "cloud": b.Cloud, "state": "terminated", "observed_at": *now, "via": "describe", "request_id": "request", "written_by": map[string]string{"box": "dead", "enrollment": b.Enrollment}} |
| + if fault == "wrong-writer" { |
| + fence["written_by"] = map[string]string{"box": "unknown", "enrollment": b.Enrollment} |
| + } |
| + raw, _ := json.Marshal(fence) |
| + if fault == "malformed" { |
| + raw = []byte("{") |
| + } |
| + if fault != "unreadable" { |
| + if err := release.CreateExact(ctx, store, "control/box-fences/"+b.Enrollment+".json", raw); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + before, err := hostclaim.Read(ctx, store, host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + wrapped := &fenceReadmitStore{Store: store, unreadable: fault == "unreadable"} |
| + s.Directory.Store = wrapped |
| + if err := s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs); err == nil { |
| + t.Fatal("fence authority ignored") |
| + } |
| + if wrapped.writes != 0 { |
| + t.Fatal("mutation before fence refusal", wrapped.writes) |
| + } |
| + after, err := hostclaim.Read(ctx, store, host) |
| + if err != nil || after.ETag != before.ETag { |
| + t.Fatal("per-host quarantine changed", err) |
| + } |
| + unchanged, err := s.Directory.Read(ctx) |
| + if err != nil || unchanged.ETag != roster.ETag { |
| + t.Fatal("global quarantine changed", err) |
| + } |
| + if unchanged.Roster.Boxes["dead"].State != "quarantined" || len(after.Claim.LossRefs) != 1 { |
| + t.Fatal("quarantine or history removed") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +type lossBookkeepingStore struct { |
| + release.Store |
| + fail bool |
| +} |
| + |
| +func (s *lossBookkeepingStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) { |
| + if s.fail && etag != "" && strings.HasPrefix(key, "control/host-loss/") { |
| + return "", errors.New("loss bookkeeping unavailable") |
| + } |
| + return s.Store.Put(ctx, key, raw, etag) |
| +} |
| +func TestPreparedLossCommitReferenceSurvivesClaimProgress(t *testing.T) { |
| + ctx := t.Context() |
| + s, store, now, host := resolveFixture(t) |
| + in := submitted(t, s, now, host) |
| + wrapped := &lossBookkeepingStore{Store: store, fail: true} |
| + s.Directory.Store = wrapped |
| + if _, err := s.Declare(ctx, "operator", host, in); err == nil { |
| + t.Fatal("bookkeeping failure not reached") |
| + } |
| + loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) |
| + if err != nil || loss.Record.State != "prepared" { |
| + t.Fatal(loss, err) |
| + } |
| + committed, err := hostclaim.Read(ctx, store, host) |
| + if err != nil || len(committed.Claim.LossRefs) != 1 || committed.Claim.LossRefs[0] != in.Confirmation { |
| + t.Fatal("claim not committed", err) |
| + } |
| + progressed, err := hostclaim.Update(ctx, store, committed, func(*hostclaim.Claim) error { return nil }) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + wrapped.fail = false |
| + if _, err := s.Declare(ctx, "operator", host, in); err != nil { |
| + t.Fatal("successful CAS treated as conflict", err) |
| + } |
| + loss, err = hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation) |
| + if err != nil || loss.Record.State != "committed" { |
| + t.Fatal(loss, err) |
| + } |
| + after, err := hostclaim.Read(ctx, store, host) |
| + if err != nil || after.ETag != progressed.ETag { |
| + t.Fatal("commit recovery changed claim", err) |
| + } |
| +} |
| +``` |
| + |
| Run this now and expect failure before the implementation is present: |
| ```sh |
| go test ./internal/hostresolve -count=1 |
| +go test ./internal/console -run '^TestHostUpdate|^TestHostRollback' -count=1 |
| go test ./internal/console -run '^TestHostClaimRoutesArePersonOperatorFlagGatedAndInert$' -count=1 |
| go test . -run '^TestHostClaimFlag|^TestParseHostResolve' -count=1 |
| ``` |
| @@ -5957,6 +6865,19 @@ func (s Service) finish(ctx context.Context, loss hostclaim.ObservedLossRecord) |
| if err != nil { |
| return hostclaim.Observed{}, err |
| } |
| + if loss.Record.State == "conflicted" { |
| + return hostclaim.Observed{}, hostclaim.ErrConflict |
| + } |
| + // A retained loss reference proves an earlier successful CAS even if |
| + // bookkeeping failed. Otherwise a changed ETag fences every stale retry. |
| + if loss.Record.State == "prepared" && (observed.Absent || !slices.Contains(observed.Claim.LossRefs, loss.Record.ID)) && (observed.Absent || observed.ETag != loss.Record.ExpectedClaimETag || observed.Claim.Gen != loss.Record.ExpectedClaimGen) { |
| + r := loss.Record |
| + r.State = "conflicted" |
| + if _, err := hostclaim.ReplaceLossRecord(ctx, s.Directory.Store, s.Signer, loss, r); err != nil { |
| + return hostclaim.Observed{}, err |
| + } |
| + return hostclaim.Observed{}, hostclaim.ErrConflict |
| + } |
| next, err := hostclaim.DeclareLost(ctx, s.Directory.Store, s.Directory.PublicKey, observed, loss.Record.ID, owner{}) |
| if err != nil { |
| return hostclaim.Observed{}, err |
| @@ -6001,6 +6922,13 @@ func (s Service) Readmit(ctx context.Context, box, enrollment string, revision u |
| if !ok || b.Enrollment != enrollment || b.State != "quarantined" || b.QuarantineRevision != revision || !reflect.DeepEqual(b.LossIDs, lossIDs) { |
| return ErrResolve |
| } |
| + fenced, err := s.Directory.TerminationFence(ctx, before, box, s.Now()) |
| + if err != nil { |
| + return err |
| + } |
| + if fenced { |
| + return ErrResolve |
| + } |
| report, err := s.Directory.PrivateBoxReport(ctx, box, b) |
| if err != nil { |
| return err |
| @@ -6039,7 +6967,7 @@ func (s Service) Readmit(ctx context.Context, box, enrollment string, revision u |
| if err != nil { |
| return err |
| } |
| - if loss.Record.State != "committed" || loss.Record.Box != box || loss.Record.Enrollment != enrollment { |
| + if (loss.Record.State != "committed" && loss.Record.State != "conflicted") || loss.Record.Box != box || loss.Record.Enrollment != enrollment { |
| return ErrResolve |
| } |
| proven := false |
| @@ -6048,7 +6976,7 @@ func (s Service) Readmit(ctx context.Context, box, enrollment string, revision u |
| proven = true |
| } |
| } |
| - if !proven && len(loss.Record.ConfirmedConsequences.DischargeLocal) == 0 { |
| + if !proven && (loss.Record.State == "conflicted" || len(loss.Record.ConfirmedConsequences.DischargeLocal) == 0) { |
| return ErrResolve |
| } |
| } |
| @@ -6197,7 +7125,9 @@ package console |
| import ( |
| "context" |
| "crypto/rand" |
| + "crypto/sha256" |
| "encoding/hex" |
| + "encoding/json" |
| "errors" |
| "github.com/carlosframework/platform/internal/hostactors" |
| "github.com/carlosframework/platform/internal/hostfloor" |
| @@ -6207,7 +7137,7 @@ import ( |
| "time" |
| ) |
| -func (s *Server) reserveHostUpdate(ctx context.Context, rel *release.SystemRelease, channel string) (string, error) { |
| +func (s *Server) reserveHostUpdate(ctx context.Context, rel *release.SystemRelease, channel, mode string, options release.SystemPromoteOptions, expectedETag ...string) (string, error) { |
| pub := release.RootSigner(s.cfg.Key).RootPublicKey() |
| floor, err := hostfloor.Read(ctx, s.cfg.Releases, pub, s.cfg.Bucket) |
| if errors.Is(err, os.ErrNotExist) { |
| @@ -6276,12 +7206,46 @@ func (s *Server) reserveHostUpdate(ctx context.Context, rel *release.SystemRelea |
| if err != nil { |
| return "", err |
| } |
| + requestRelease := *rel |
| + requestRelease.ReleasedAt = time.Time{} |
| + request, err := json.Marshal(struct { |
| + Channel string |
| + Mode string |
| + Release release.SystemRelease |
| + Spread time.Duration |
| + }{channel, mode, requestRelease, options.Spread}) |
| + if err != nil { |
| + return "", err |
| + } |
| + digest := sha256.Sum256(request) |
| + requestID := hex.EncodeToString(digest[:]) |
| + for id, existing := range o.Roster.Activations { |
| + if existing.Publication == nil || existing.Publication.Channel != channel { |
| + continue |
| + } |
| + if existing.RequestSHA256 != requestID || existing.FloorSHA256 != floor.Digest { |
| + return "", hostactors.ErrRoster |
| + } |
| + if _, err := existing.Publication.Verify(pub); err != nil { |
| + return "", err |
| + } |
| + return id, nil |
| + } |
| var nonce [16]byte |
| if _, err = rand.Read(nonce[:]); err != nil { |
| return "", err |
| } |
| id := hex.EncodeToString(nonce[:]) |
| - h := hostactors.Reservation{ID: id, FloorSHA256: floor.Digest, Actors: map[string]string{}, Releases: map[string]hostactors.Report{}, Hashes: map[string][]string{}} |
| + var publication release.SystemPublication |
| + if mode == "rollback" { |
| + publication, err = release.PrepareSystemRollback(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), "router", channel, options.Actor, id, expectedETag...) |
| + } else { |
| + publication, err = release.PrepareSystemPromotion(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), "router", channel, *rel, options, id) |
| + } |
| + if err != nil { |
| + return "", err |
| + } |
| + h := hostactors.Reservation{Publication: &publication, RequestSHA256: requestID, UpdateMode: mode, ID: id, FloorSHA256: floor.Digest, Actors: map[string]string{}, Releases: map[string]hostactors.Report{}, Hashes: map[string][]string{}} |
| e := hostactors.ExactEvidence{PublicKey: d.PublicKey, Store: s.cfg.Releases, Boxes: d.PrivateBoxReport} |
| for ref, b := range o.Roster.Boxes { |
| if b.State != "active" { |
| @@ -6328,6 +7292,57 @@ func (s *Server) reserveHostUpdate(ctx context.Context, rel *release.SystemRelea |
| _, err = d.Reserve(ctx, release.RootSigner(s.cfg.Key), o, h, true) |
| return id, err |
| } |
| + |
| +func (s *Server) promoteHostUpdate(ctx context.Context, rel release.SystemRelease, component, channel string, options release.SystemPromoteOptions) (*release.SystemPointer, error) { |
| + id, err := s.reserveHostUpdate(ctx, &rel, channel, "promote", options) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if id == "" { |
| + return release.PromoteSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), component, channel, rel, options) |
| + } |
| + d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: release.RootSigner(s.cfg.Key).RootPublicKey(), Bucket: s.cfg.Bucket} |
| + return d.PublishActivation(ctx, id) |
| +} |
| + |
| +func (s *Server) rollbackHostUpdate(ctx context.Context, component, channel, actor string) (*release.SystemPointer, error) { |
| + d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: release.RootSigner(s.cfg.Key).RootPublicKey(), Bucket: s.cfg.Bucket} |
| + floor, err := hostfloor.Read(ctx, s.cfg.Releases, d.PublicKey, s.cfg.Bucket) |
| + if err == nil { |
| + roster, err := d.Read(ctx) |
| + if err != nil { |
| + return nil, err |
| + } |
| + // A successful publication with a lost response must not pop history again. |
| + for id, h := range roster.Roster.Activations { |
| + if h.Publication == nil || h.Publication.Channel != channel { |
| + continue |
| + } |
| + if h.UpdateMode != "rollback" || h.FloorSHA256 != floor.Digest { |
| + return nil, hostactors.ErrRoster |
| + } |
| + return d.PublishActivation(ctx, id) |
| + } |
| + } else if !errors.Is(err, os.ErrNotExist) { |
| + return nil, err |
| + } |
| + current, etag, err := release.GetSystemPointer(ctx, s.cfg.Releases, d.PublicKey, component, channel) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if len(current.History) == 0 { |
| + return nil, release.ErrNoSystemHistory |
| + } |
| + target := current.History[0] |
| + id, err := s.reserveHostUpdate(ctx, &target, channel, "rollback", release.SystemPromoteOptions{Actor: actor}, etag) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if id == "" { |
| + return release.RollbackSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), component, channel, actor, etag) |
| + } |
| + return d.PublishActivation(ctx, id) |
| +} |
| ``` |
| **Step 3: apply the exact existing-file patch.** |
| @@ -6467,21 +7482,12 @@ func (s *Server) reserveHostUpdate(ctx context.Context, rel *release.SystemRelea |
| + |
| --- a/internal/console/apisystemupdate.go |
| +++ b/internal/console/apisystemupdate.go |
| -@@ -461,0 +462,3 @@ |
| -+ if _, err := s.reserveHostUpdate(ctx, &rel, in.Channel); err != nil { |
| -+ return err |
| -+ } |
| -@@ -707 +710,9 @@ |
| +@@ -462 +462 @@ |
| +- p, err := release.PromoteSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), in.Component, in.Channel, rel, |
| ++ p, err := s.promoteHostUpdate(ctx, rel, in.Component, in.Channel, |
| +@@ -707 +707 @@ |
| - p, err := release.RollbackSystem(r.Context(), s.cfg.Releases, release.RootSigner(s.cfg.Key), in.Component, in.Channel, addr) |
| -+ current, currentETag, readErr := release.GetSystemPointer(r.Context(), s.cfg.Releases, release.RootSigner(s.cfg.Key).RootPublicKey(), in.Component, in.Channel) |
| -+ if readErr == nil && len(current.History) > 0 { |
| -+ target := current.History[0] |
| -+ if _, err := s.reserveHostUpdate(r.Context(), &target, in.Channel); err != nil { |
| -+ jsonError(w, http.StatusConflict, err.Error()) |
| -+ return |
| -+ } |
| -+ } |
| -+ p, err := release.RollbackSystem(r.Context(), s.cfg.Releases, release.RootSigner(s.cfg.Key), in.Component, in.Channel, addr, currentETag) |
| ++ p, err := s.rollbackHostUpdate(r.Context(), in.Component, in.Channel, addr) |
| --- a/cmd/console/main.go |
| +++ b/cmd/console/main.go |
| @@ -298,0 +299 @@ |
| @@ -6576,6 +7582,7 @@ func (s *Server) reserveHostUpdate(ctx context.Context, rel *release.SystemRelea |
| ```sh |
| go test ./internal/hostresolve -count=1 |
| +go test ./internal/console -run '^TestHostUpdate|^TestHostRollback' -count=1 |
| go test ./internal/console -run '^TestHostClaimRoutesArePersonOperatorFlagGatedAndInert$' -count=1 |
| go test . -run '^TestHostClaimFlag|^TestParseHostResolve' -count=1 |
| ``` |
| @@ -6845,11 +7852,11 @@ or cloud resource was changed while writing this plan. |
| ## Authoring verification |
| -Rev 3 was reconstructed from this document's 42 complete Go files and five |
| -patches covering 12 existing files in a disposable export of |
| -`ca5a79d8a3821172f5829231014969f106bc33e9`. Every printed Go file matched |
| -its compiled source byte for byte, and every patch applied to that parent. |
| -No implementation files or branch were created in the design worktree. |
| +Rev 4 was reconstructed from this document's 47 complete Go files and five |
| +patches covering 12 existing files in a fresh disposable export of |
| +`ca5a79d8a3821172f5829231014969f106bc33e9`. Every printed Go file and patched |
| +file matched its tested source byte for byte. All patches applied to that |
| +parent. No implementation files or branch were created in the design worktree. |
| These checks passed on the final reconstruction: |
| @@ -6857,39 +7864,53 @@ These checks passed on the final reconstruction: |
| go build -buildvcs=false ./... |
| go test -buildvcs=false -exec=/bin/true ./... |
| go test -buildvcs=false ./internal/hostactors ./internal/hostfloor ./internal/hostresolve ./internal/hostfence ./internal/hostexec -count=1 |
| -go test -buildvcs=false ./internal/host -run 'TestFloorRefusal|TestGuardRecovery|TestUnpublishedCompletion' -count=1 |
| +go test -buildvcs=false ./internal/release -run 'TestSystemPublication|TestSystemPointer|TestSystemPromote|TestSystemRollback' -count=1 |
| +go test -buildvcs=false ./internal/host ./internal/console -run 'TestFloorRefusal|TestGuard|TestUnpublishedCompletion|TestHostUpdate|TestHostRollback|TestHostClaimRoutesArePersonOperatorFlagGatedAndInert' -count=1 |
| gofmt -l . |
| ``` |
| `-exec=/bin/true` compiles every test package without executing its tests. |
| -The runtime checks above are targeted. The full repository runtime gate, |
| -race checks and cross-platform builds remain implementation-branch gates; |
| -this revision does not claim to have rerun rev 2's broader checks. |
| - |
| -The round 2 regressions are: |
| - |
| -- `TestStartRevalidatesReservedActorEligibility`: reserve, then quarantine, |
| - retire or terminate a participant or bound console compute box. Start |
| - refuses before invoking the transition and leaves the claim unchanged. |
| - Malformed and unreadable fence authority also refuse. |
| -- `TestFleetInvitationBindsReferenceToEnrollment`: generate enrollment, |
| - derive the fleet reference, bind the unchanged signed cloud evidence, |
| - retry the invitation and complete enrollment. Arbitrary references and |
| - mismatched fleet evidence refuse. |
| -- `TestGuardRecoveryRefusesControllersAfterBegin`: pending, active, lost, |
| - malformed and unreadable controller state after the initial update scan |
| - refuse journaled recovery and both unfinished no-journal paths. No binary, |
| - journal, marker or completion change occurs. |
| -- `TestDeclareCrashAndChallengeExpiryAllowRetryAndReadmission`: interrupt |
| - before or after quarantine and before loss creation, expire the challenge, |
| - retry from durable acceptance, then complete readmission. The original |
| - acceptance and fence times survive. Failure to persist acceptance leaves |
| - no quarantine, and an unaccepted expired submission still refuses. An |
| - unrelated roster CAS does not strand accepted evidence. |
| - |
| -Removing each corresponding fix was watched fail its behavioral regression. |
| -Every mutation was removed before the final build and runtime checks. The |
| -Markdown passes `git diff --check`, has balanced fences and no em dashes. |
| +The runtime checks above are targeted. A broader `internal/release` runtime |
| +attempt stopped at `TestConditionalDeleteS3ActualWire`: the sandbox refused |
| +its local listener with `socket: operation not permitted`. The focused release |
| +checks passed afterward. The full repository runtime gate, race checks and |
| +cross-platform builds remain implementation-branch gates. |
| + |
| +The round 3 regressions are: |
| + |
| +- `TestPreparedLossClaimProgressAllowsRetiredReadmission`: fail the claim |
| + write after quarantine, advance the claim with an intervening retirement |
| + CAS, retry and persist `conflicted`. The accepted evidence stays unchanged, |
| + stale consequences are not applied, and readmission succeeds only with |
| + retirement evidence for that exact installation. Loss history survives. |
| +- `TestPreparedLossCommitReferenceSurvivesClaimProgress`: commit the claim, |
| + fail loss bookkeeping, advance the claim again, then retry. The retained |
| + claim reference completes committed bookkeeping instead of recording a |
| + false conflict. |
| +- `TestReadmitRefusesTerminationAuthorityBeforeMutation`: valid termination, |
| + malformed evidence, a mismatched writer and unreadable evidence each refuse |
| + with zero writes. Both per-host and global quarantine remain unchanged. |
| +- `TestHostUpdateReservationRetriesPublication`: reserve, then crash or fail |
| + promotion, restart the console and retry successfully with the same ID and |
| + signed bytes. A committed PUT with a lost response is reconciled by exact |
| + readback; unreadable reconciliation retains the reservation for retry. |
| +- `TestHostRollbackResumesWithoutPoppingAgain`: a failed rollback publication |
| + resumes its stored target; retry after success does not pop history again. |
| +- `TestHostUpdateCancellationBlocksDelayedPublication`, |
| + `TestSystemPublicationCancellationWinsDelayedCAS` and |
| + `TestSystemPublicationCancellationCannotUndoCommittedUpdate`: cancellation |
| + fences delayed publication, including an originally absent pointer, and |
| + cannot release an activation whose publication already won. |
| +- `TestGuardRejectsOldPointerForUnpublishedActivation`: an older pointer for |
| + the same target hash cannot consume a publication-backed reservation. Root |
| + accepts it only after the exact activation ID is published. |
| + |
| +The loss-progress and termination-readmission regressions failed against |
| +rev 3 before their fixes. Removing conflicted readmission, its retirement |
| +requirement, reservation reuse or root's activation-ID check was also watched |
| +fail the corresponding behavioral regression. Every mutation was restored |
| +before the final reconstruction and checks. The Markdown passes |
| +`git diff --check`, has balanced fences and contains no em dashes. |
| The default Go cache was attempted and was read-only in this sandbox. |
| Verification reused the existing shared on-disk cache with |
| @@ -6897,10 +7918,10 @@ Verification reused the existing shared on-disk cache with |
| used no RAM-backed `/tmp` scratch. Disposable exports and authoring scratch |
| were removed after verification. The shared cache was retained. |
| -An independent GPT-6 Astra review checked the four authority-related fixes. |
| -Its finding about acceptance retry after an unrelated roster CAS was fixed |
| -and covered by the crash regression. Opus was unavailable through the |
| -collaboration tool, so this does not replace the repository's required Opus |
| -review before implementation lands. Cross-branch activation dependencies |
| -remain gates. Only this Markdown plan changed; nothing was staged or |
| -committed, and no production resource, floor, flag or service changed. |
| +An independent GPT-6 Astra review checked all three authority-related fixes. |
| +Its finding about an older same-target pointer consuming an unpublished |
| +activation was fixed and covered by the guard regression. Focused re-review |
| +found no remaining issue. Opus was unavailable through the collaboration |
| +tool, so this does not replace the repository's required Opus review before |
| +implementation lands. Only this Markdown plan changed; nothing was staged |
| +or committed, and no production resource, floor, flag or service changed. |