amadan

carlos / platform Public

Merge origin/main (3407d114, claim C5/C7) into moves-phase1

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 804623c23829eb6ca4023874770f846a377a3aa6 parent e801e31parent 3407d11
36 files changed, +3987 −24
  • cmd/console/main.go +4 −0
  • internal/console/hostclaimbackfill.go +65 −0
  • internal/console/hostclaimbackfill_test.go +84 −0
  • internal/console/hostclaimops.go +63 −0
  • internal/console/hostclaimops_test.go +31 −0
  • internal/console/server.go +4 −1
  • internal/domains/claims.go +10 −7
  • internal/host/agent.go +6 −2
  • internal/host/claimbackfill.go +32 −0
  • internal/host/claimbackfill_test.go +230 −0
  • internal/hostactors/censuskey.go +59 −0
  • internal/hostactors/roster.go +15 −11
  • internal/hostclaim/adopt.go +1 −1
  • internal/hostclaim/claim.go +5 −1
  • internal/hostclaim/create.go +2 −1
  • internal/hostclaim/fence.go +109 −0
  • internal/hostclaim/relocating.go +142 −0
  • internal/hostclaim/relocating_test.go +251 −0
  • internal/hostfence/claim_fence.go +38 −0
  • internal/hostfence/claim_fence_test.go +72 −0
  • internal/hostops/backfill.go +380 −0
  • internal/hostops/backfill_controller.go +123 −0
  • internal/hostops/backfill_publish.go +162 −0
  • internal/hostops/backfill_test.go +357 −0
  • internal/hostops/operations.go +304 −0
  • internal/hostops/operations_test.go +629 −0
  • internal/hostops/purge.go +196 −0
  • internal/hostops/record.go +182 −0
  • internal/hostops/recovery.go +216 −0
  • internal/hostops/remote.go +47 −0
  • internal/hostops/retirement.go +69 −0
  • internal/provision/claim_export.go +3 −0
  • internal/provision/provision.go +1 −0
  • internal/provision/retirement.go +53 −0
  • internal/provision/retirement_test.go +41 −0
  • internal/provision/sidecar.go +1 −0
diff --git a/cmd/console/main.go b/cmd/console/main.go
index c02393d23..11923395b 100644
--- a/cmd/console/main.go
+++ b/cmd/console/main.go
@@ -429,6 +429,10 @@ func serve() {
// objects and pulls the AWS bill (internal/console/economicspass.go).
// Same placement rule as the alert watcher: started here, never in
// NewServer, so constructing a Server stays goroutine-free.
+ if err := srv.ConfigureClaimBackfill(); err != nil {
+ log.Printf("console: claim recovery: %v", err)
+ }
+ go srv.StartClaimRecoveryPass(context.Background())
go srv.StartEconomicsPass(context.Background())
// The infrastructure inventory pass writes infra/inventory.json for
// /fleet/architecture (internal/console/infrapass.go). Same placement
diff --git a/internal/console/hostclaimbackfill.go b/internal/console/hostclaimbackfill.go
new file mode 100644
index 000000000..92fc5881b
--- /dev/null
+++ b/internal/console/hostclaimbackfill.go
@@ -0,0 +1,65 @@
+package console
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "errors"
+ "log"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostops"
+)
+
+func claimOperationID() string {
+ var b [16]byte
+ if _, err := rand.Read(b[:]); err != nil {
+ return ""
+ }
+ return hex.EncodeToString(b[:])
+}
+
+// Called after NewServer, before background workers. It starts no goroutine.
+func (s *Server) ConfigureClaimBackfill() error {
+ if !s.cfg.HostClaim {
+ return nil
+ }
+ if s.cfg.Releases == nil || s.cfg.Bucket == "" || len(s.cfg.ReleasePub) == 0 {
+ s.cfg.HostClaimOps = nil
+ log.Printf("console: claim recovery: %v", hostops.ErrContract)
+ return nil
+ }
+ if s.cfg.HostClaimOps == nil {
+ d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: s.cfg.ReleasePub, Bucket: s.cfg.Bucket}
+ c := hostops.NewBackfillController(s.cfg.Releases, d, time.Now, claimOperationID)
+ s.cfg.HostClaimOps = &c
+ }
+ return nil
+}
+func (s *Server) ReconcileHostClaims(ctx context.Context) error {
+ if !s.cfg.HostClaim {
+ return nil
+ }
+ if s.cfg.HostClaimOps == nil {
+ return hostops.ErrContract
+ }
+ return hostops.ReconcileClaims(ctx, *s.cfg.HostClaimOps, s.adoptHostRecord)
+}
+
+// Exported for the moves console pass; construction and startup remain separate.
+func (s *Server) RunClaimRecoveryPass(ctx context.Context) error {
+ if !s.cfg.HostClaim {
+ return nil
+ }
+ var errs []error
+ if err := s.ReconcileHostClaims(ctx); err != nil {
+ errs = append(errs, err)
+ }
+ if s.cfg.HostClaimRecovery != nil {
+ if err := s.cfg.HostClaimRecovery.Pass(ctx); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ return errors.Join(errs...)
+}
diff --git a/internal/console/hostclaimbackfill_test.go b/internal/console/hostclaimbackfill_test.go
new file mode 100644
index 000000000..e51440e14
--- /dev/null
+++ b/internal/console/hostclaimbackfill_test.go
@@ -0,0 +1,84 @@
+package console
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "log"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func TestConfigureClaimBackfillMissingContractDoesNotFailStartup(t *testing.T) {
+ for _, missing := range []string{"store", "bucket", "public-key"} {
+ t.Run(missing, func(t *testing.T) {
+ s := &Server{cfg: Config{HostClaim: true, Releases: release.NewDirStore(t.TempDir()), Bucket: "bucket", ReleasePub: make(ed25519.PublicKey, ed25519.PublicKeySize)}}
+ switch missing {
+ case "store":
+ s.cfg.Releases = nil
+ case "bucket":
+ s.cfg.Bucket = ""
+ case "public-key":
+ s.cfg.ReleasePub = nil
+ }
+ var logs bytes.Buffer
+ previous := log.Writer()
+ log.SetOutput(&logs)
+ t.Cleanup(func() { log.SetOutput(previous) })
+ if err := s.ConfigureClaimBackfill(); err != nil {
+ t.Errorf("configuration failed startup: %v", err)
+ }
+ if s.cfg.HostClaimOps != nil || !strings.Contains(logs.String(), hostops.ErrContract.Error()) {
+ t.Fatalf("ops=%v log=%q", s.cfg.HostClaimOps, logs.String())
+ }
+ s.StartClaimRecoveryPass(context.Background())
+ })
+ }
+}
+
+func TestClaimBackfillQuietWithoutAdoptingPlan(t *testing.T) {
+ for _, phase := range []string{"absent", "collecting", "done", "future"} {
+ t.Run(phase, func(t *testing.T) {
+ pub, key, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ store := release.NewDirStore(t.TempDir())
+ if phase != "absent" {
+ raw, err := release.SignRootEnvelope(release.RootSigner(key), hostops.BackfillPlan{V: 1, ID: "backfill", Phase: phase, CreatedAt: time.Now()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := store.Put(context.Background(), hostops.BackfillKey, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+ s := &Server{cfg: Config{HostClaim: true, Releases: store, Bucket: "bucket", ReleasePub: pub}}
+ if err := s.ConfigureClaimBackfill(); err != nil {
+ t.Fatal(err)
+ }
+ ops := s.cfg.HostClaimOps
+ if ops == nil {
+ t.Fatal("backfill not configured")
+ }
+ if err := s.ConfigureClaimBackfill(); err != nil || s.cfg.HostClaimOps != ops {
+ t.Fatal("controller replaced", err)
+ }
+ var logs bytes.Buffer
+ previous := log.Writer()
+ log.SetOutput(&logs)
+ t.Cleanup(func() { log.SetOutput(previous) })
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ s.StartClaimRecoveryPass(ctx)
+ if logs.Len() != 0 {
+ t.Fatalf("inactive backfill logged: %s", &logs)
+ }
+ })
+ }
+}
diff --git a/internal/console/hostclaimops.go b/internal/console/hostclaimops.go
new file mode 100644
index 000000000..ef9aa7647
--- /dev/null
+++ b/internal/console/hostclaimops.go
@@ -0,0 +1,63 @@
+package console
+
+import (
+ "context"
+ "log"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+func (s *Server) retireHostRecord(ctx context.Context, ref provision.RetireRef) error {
+ var claims provision.ClaimRetirer
+ if s.cfg.HostClaimOps != nil {
+ claims = s.cfg.HostClaimOps
+ }
+ return provision.RetireRecord(ctx, s.cfg.Releases, ref, provision.RetireOptions{HostClaim: s.cfg.HostClaim, Claims: claims})
+}
+func (s *Server) adoptHostRecord(ctx context.Context, key string) (hostclaim.Observed, error) {
+ if !s.cfg.HostClaim || s.cfg.HostClaimOps == nil {
+ return hostclaim.Observed{}, hostops.ErrContract
+ }
+ rec, err := hostops.ReadRecord(ctx, s.cfg.Releases, key)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ return s.cfg.HostClaimOps.Adopt(ctx, rec)
+}
+
+const claimRecoveryInterval = 2 * time.Minute
+
+// Called by serve, never NewServer. Tests can invoke the one-pass method.
+func (s *Server) StartClaimRecoveryPass(ctx context.Context) {
+ if !s.cfg.HostClaim {
+ return
+ }
+ if s.cfg.HostClaimOps == nil {
+ log.Printf("console: claim recovery: %v", hostops.ErrContract)
+ return
+ }
+ t := time.NewTicker(claimRecoveryInterval)
+ defer t.Stop()
+ for {
+ if err := s.RunClaimRecoveryPass(ctx); err != nil {
+ log.Printf("console: claim recovery: %v", err)
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case <-t.C:
+ }
+ }
+}
+func (s *Server) claimRecoveryPass(ctx context.Context) error {
+ if !s.cfg.HostClaim {
+ return nil
+ }
+ if s.cfg.HostClaimRecovery == nil {
+ return hostops.ErrContract
+ }
+ return s.cfg.HostClaimRecovery.Pass(ctx)
+}
diff --git a/internal/console/hostclaimops_test.go b/internal/console/hostclaimops_test.go
new file mode 100644
index 000000000..841d8f0f4
--- /dev/null
+++ b/internal/console/hostclaimops_test.go
@@ -0,0 +1,31 @@
+package console
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func TestClaimRecoveryAndRetirementAreInert(t *testing.T) {
+ s := &Server{cfg: Config{Releases: release.NewDirStore(t.TempDir())}}
+ ctx := context.Background()
+ if err := s.claimRecoveryPass(ctx); err != nil {
+ t.Fatal(err)
+ }
+ s.StartClaimRecoveryPass(ctx) // Must return without a ticker or store reads.
+ ref := provision.RetireRef{Family: "instance", Account: "bab", App: "jam", Host: "shop.bab.example.com"}
+ if err := s.retireHostRecord(ctx, ref); err != nil {
+ t.Fatal(err)
+ }
+ s.cfg.HostClaim = true
+ if err := s.claimRecoveryPass(ctx); !errors.Is(err, hostops.ErrContract) {
+ t.Fatal(err)
+ }
+ if err := s.retireHostRecord(ctx, ref); !errors.Is(err, provision.ErrClaimRetirer) {
+ t.Fatal(err)
+ }
+}
diff --git a/internal/console/server.go b/internal/console/server.go
index a6e495272..56c3cbd40 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -14,6 +14,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "github.com/carlosframework/platform/internal/hostops"
"github.com/carlosframework/platform/internal/lease"
"html/template"
"log"
@@ -470,7 +471,9 @@ const (
// Config is everything the server reads its world through.
type Config struct {
- HostConsoleID string
+ HostClaimOps *hostops.Controller
+ HostClaimRecovery *hostops.Recovery
+ HostConsoleID string
HostClaim bool
DBVersions lease.VersionedStore
diff --git a/internal/domains/claims.go b/internal/domains/claims.go
index a31cfadb2..000528466 100644
--- a/internal/domains/claims.go
+++ b/internal/domains/claims.go
@@ -35,13 +35,16 @@ func ClaimKey(host string) string { return ClaimsPrefix + host + ".json" }
// CreatedBy is omitempty because a claim minted by a box-side process, not a
// human door, has no operator to name.
type Claim struct {
- Schema int `json:"schema"` // 1
- Host string `json:"host"`
- Account string `json:"account"`
- App string `json:"app"`
- Route string `json:"route,omitempty"`
- CreatedAt string `json:"created_at"`
- CreatedBy string `json:"created_by,omitempty"`
+ Acquisition string `json:"acq,omitempty"`
+ RecordIncarnation string `json:"incarnation,omitempty"`
+ OwnershipState string `json:"ownership_state,omitempty"`
+ Schema int `json:"schema"` // 1
+ Host string `json:"host"`
+ Account string `json:"account"`
+ App string `json:"app"`
+ Route string `json:"route,omitempty"`
+ CreatedAt string `json:"created_at"`
+ CreatedBy string `json:"created_by,omitempty"`
// Wildcard and Catchall are the woodstar-carloku branch's two claim
// capabilities (its migration spec §4-5 gives them meaning; nothing in
// this package acts on either). They are deliberately orthogonal:
diff --git a/internal/host/agent.go b/internal/host/agent.go
index ef5f97bbd..bd96fc692 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -551,7 +551,7 @@ func runAgent(args []string, buildClaims ClaimFactory) {
// by CARLOS_HOST_METRICS — one log line and nothing else where unset.
go memwatchLoop()
go domainsLoop(reg, peekAcmeCache(args))
- go instancesLoop(reg)
+ go instancesLoop(reg, claims.Writes)
// The deployment-global election, armed BEFORE the loops it gates so a
// single-box deployment is holding well before their first tick (a
// minute away) — see housekeeper.go. Not a `go` of its own: it publishes
@@ -2772,7 +2772,7 @@ func domainsDuty(pass func(context.Context) error) housekeeperDuty {
// already had a store+reg in scope) — that coupled binding materialization
// to PLATFORM_APPS_DOMAIN being set, which has nothing to do with fleets;
// this loop has no such gate and shares the same store already opened above.
-func instancesLoop(reg *registry.Registry) {
+func instancesLoop(reg *registry.Registry, writes hostwrite.Config) {
ctx := context.Background()
store, err := openDeployStore(ctx)
if err != nil {
@@ -2780,6 +2780,7 @@ func instancesLoop(reg *registry.Registry) {
return
}
cfg := provisionTickConfig(os.Getenv)
+ backfill := newClaimBackfill(writes, store, cfg.ReleasePub, os.Getenv("CARLOS_DEPLOYMENT_BUCKET"))
log.Printf("agent: instances loop armed (5min + jitter), region %q pools %v", cfg.Region, cfg.Pools)
// Platform-only: a fleet box (CARLOS_FLEET set) never reads other
// fleets' bindings — it only ever WRITES its own (fleetbind.go's
@@ -2793,6 +2794,9 @@ func instancesLoop(reg *registry.Registry) {
for {
time.Sleep(delay)
delay = 5*time.Minute + rand.N(time.Minute)
+ if err := backfill(ctx); err != nil {
+ log.Printf("agent: claim backfill: %v", err)
+ }
if err := reconcileSerialized(ctx, store, reg, cfg); err != nil {
log.Printf("agent: instances pass: %v", err)
}
diff --git a/internal/host/claimbackfill.go b/internal/host/claimbackfill.go
new file mode 100644
index 000000000..51f7e7a2d
--- /dev/null
+++ b/internal/host/claimbackfill.go
@@ -0,0 +1,32 @@
+package host
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "encoding/hex"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func newClaimBackfill(writes hostwrite.Config, s release.Store, pub ed25519.PublicKey, bucket string) func(context.Context) error {
+ if !writes.Enabled {
+ return func(context.Context) error { return nil }
+ }
+ if s == nil || len(pub) == 0 || bucket == "" {
+ return func(context.Context) error { return hostops.ErrContract }
+ }
+ d := hostactors.Directory{Store: s, PublicKey: pub, Bucket: bucket}
+ c := hostops.NewBackfillController(s, d, time.Now, func() string {
+ var b [16]byte
+ if _, err := rand.Read(b[:]); err != nil {
+ return ""
+ }
+ return hex.EncodeToString(b[:])
+ })
+ return c.ReconcileClaims
+}
diff --git a/internal/host/claimbackfill_test.go b/internal/host/claimbackfill_test.go
new file mode 100644
index 000000000..c2d57e2a9
--- /dev/null
+++ b/internal/host/claimbackfill_test.go
@@ -0,0 +1,230 @@
+package host
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/domains"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func TestClaimBackfillRunsOnlyOnInstancesCadence(t *testing.T) {
+ file, err := parser.ParseFile(token.NewFileSet(), "agent.go", nil, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, decl := range file.Decls {
+ fn, ok := decl.(*ast.FuncDecl)
+ if !ok {
+ continue
+ }
+ switch fn.Name.Name {
+ case "fastConvergeLoop":
+ ast.Inspect(fn.Body, func(n ast.Node) bool {
+ call, ok := n.(*ast.CallExpr)
+ if !ok {
+ return true
+ }
+ id, ok := call.Fun.(*ast.Ident)
+ if ok && (id.Name == "reconcileBackfill" || id.Name == "newClaimBackfill") {
+ t.Error("backfill runs on the serial promote path")
+ }
+ return true
+ })
+ case "instancesLoop":
+ constructors := 0
+ for _, stmt := range fn.Body.List {
+ loop, inLoop := stmt.(*ast.ForStmt)
+ var node ast.Node = stmt
+ if inLoop {
+ node = loop.Body
+ }
+ ast.Inspect(node, func(n ast.Node) bool {
+ call, ok := n.(*ast.CallExpr)
+ if !ok {
+ return true
+ }
+ id, ok := call.Fun.(*ast.Ident)
+ if ok && id.Name == "newClaimBackfill" {
+ constructors++
+ if inLoop {
+ t.Error("backfill controller constructed per tick")
+ }
+ }
+ return true
+ })
+ }
+ if constructors != 1 {
+ t.Errorf("instances loop has %d persistent backfill constructors, want 1", constructors)
+ }
+ }
+ }
+}
+
+type backfillPassStore struct {
+ release.Store
+ gets, censusReads int
+ failClaim error
+}
+
+func (s *backfillPassStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ s.gets++
+ if strings.HasPrefix(key, "control/host-census/") {
+ s.censusReads++
+ }
+ return s.Store.Get(ctx, key)
+}
+
+func (s *backfillPassStore) Put(ctx context.Context, key string, body []byte, tag string) (string, error) {
+ if strings.HasPrefix(key, "control/hosts/") && s.failClaim != nil {
+ err := s.failClaim
+ s.failClaim = nil
+ return "", err
+ }
+ return s.Store.Put(ctx, key, body, tag)
+}
+
+func TestClaimBackfillPassInertAndMissingContract(t *testing.T) {
+ store := &backfillPassStore{Store: release.NewDirStore(t.TempDir())}
+ ctx := context.Background()
+ if err := newClaimBackfill(hostwrite.Config{}, store, nil, "")(ctx); err != nil || store.gets != 0 {
+ t.Fatal("flag off accessed backfill", err, store.gets)
+ }
+ for _, missing := range []string{"store", "key", "bucket"} {
+ var s release.Store = store
+ pub := make(ed25519.PublicKey, ed25519.PublicKeySize)
+ bucket := "bucket"
+ switch missing {
+ case "store":
+ s = nil
+ case "key":
+ pub = nil
+ case "bucket":
+ bucket = ""
+ }
+ if err := newClaimBackfill(hostwrite.Config{Enabled: true}, s, pub, bucket)(ctx); !errors.Is(err, hostops.ErrContract) || store.gets != 0 {
+ t.Fatalf("missing %s: err=%v reads=%d", missing, err, store.gets)
+ }
+ }
+}
+
+func TestClaimBackfillPassKeepsCensusBetweenRetries(t *testing.T) {
+ ctx := context.Background()
+ store := &backfillPassStore{Store: release.NewDirStore(t.TempDir())}
+ pub, key, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ boxPub, boxKey, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ now := time.Now().UTC()
+ put := func(path string, value any, signed bool) {
+ t.Helper()
+ raw, err := json.Marshal(value)
+ if signed {
+ raw, err = release.SignRootEnvelope(release.RootSigner(key), value)
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := store.Put(ctx, path, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+ enrollment := strings.Repeat("a", 32)
+ roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1,
+ Regions: map[string]hostactors.Region{"ie": {}},
+ Boxes: map[string]hostactors.Box{"box-a": {CensusPublicKey: boxPub, Enrollment: enrollment, Namespace: "platform", BoxID: "box-a", Label: "box-a", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "ie", InstanceID: "i-box-a"}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"}},
+ Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
+ put(hostactors.Key, roster, true)
+ r, err := hostops.PrepareInstance(provision.Record{Schema: 1, Account: "bab", App: "jam", Host: "legacy.example.com", Slug: "shop", Channel: "edge", Region: "ie", Acquisition: "acquisition"}, now, "ie")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := store.Put(ctx, r.Holder.Key, r.Body, ""); err != nil {
+ t.Fatal(err)
+ }
+ r, err = hostops.ReadRecord(ctx, store, r.Holder.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ inc, err := r.Incarnation()
+ if err != nil {
+ t.Fatal(err)
+ }
+ put(domains.ClaimKey(r.Holder.Host), domains.Claim{Schema: 1, Host: r.Holder.Host, Account: "bab", App: "jam", Acquisition: "acquisition", RecordIncarnation: inc, OwnershipState: "committed"}, false)
+ report := hostops.CensusReport{Box: "box-a", Enrollment: enrollment, BootID: "boot", Reservation: "backfill", DirectoryRevision: 1, CapturedAt: now, Complete: true, Installations: []hostops.CensusInstallation{{Host: r.Holder.Host, Account: "bab", App: "jam", Family: "instance", Key: r.Holder.Key, Incarnation: inc, Local: true, Route: true}}}
+ canonical, err := json.Marshal(report)
+ if err != nil {
+ t.Fatal(err)
+ }
+ reportPath := "control/host-census/backfill/box-a.json"
+ put(reportPath, hostops.CensusEnvelope{Report: report, Signature: ed25519.Sign(boxKey, canonical)}, false)
+ raw, _, err := store.Get(ctx, reportPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ digest := sha256.Sum256(raw)
+ put(hostops.BackfillKey, hostops.BackfillPlan{V: 1, ID: "backfill", Phase: "adopting", DirectoryRevision: 1, CreatedAt: now, Reports: map[string]string{"box-a": hex.EncodeToString(digest[:])}, Hosts: map[string]hostops.BackfillHost{r.Holder.Host: {Key: r.Holder.Key, ETag: r.ETag, Acquisition: "acquisition", Region: "ie", Box: "box-a"}}}, true)
+ store.censusReads = 0
+ pass := newClaimBackfill(hostwrite.Config{Enabled: true}, store, pub, "bucket")
+ fail := errors.New("claim write failed")
+ store.failClaim = fail
+ if err := pass(ctx); !errors.Is(err, fail) {
+ t.Fatalf("first pass: %v", err)
+ }
+ if err := pass(ctx); err != nil {
+ t.Fatalf("retry: %v", err)
+ }
+ if store.censusReads != 1 {
+ t.Fatalf("census reads=%d, want one across both passes", store.censusReads)
+ }
+ obs, err := hostclaim.Read(ctx, store, r.Holder.Host)
+ if err != nil || obs.Absent || obs.Claim.Holder.Acquisition != "acquisition" || obs.Claim.Op.Exec != "box-a" {
+ t.Fatalf("retry did not adopt: %+v %v", obs, err)
+ }
+}
+
+func TestClaimBackfillPassQuietWithoutAdoptingPlan(t *testing.T) {
+ for _, phase := range []string{"absent", "collecting", "done", "future"} {
+ t.Run(phase, func(t *testing.T) {
+ ctx := context.Background()
+ pub, key, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ store := &backfillPassStore{Store: release.NewDirStore(t.TempDir())}
+ if phase != "absent" {
+ raw, err := release.SignRootEnvelope(release.RootSigner(key), hostops.BackfillPlan{V: 1, ID: "backfill", Phase: phase, CreatedAt: time.Now()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := store.Put(ctx, hostops.BackfillKey, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+ pass := newClaimBackfill(hostwrite.Config{Enabled: true}, store, pub, "bucket")
+ if err := pass(ctx); err != nil || store.gets != 1 {
+ t.Fatalf("inactive backfill: err=%v reads=%d", err, store.gets)
+ }
+ })
+ }
+}
diff --git a/internal/hostactors/censuskey.go b/internal/hostactors/censuskey.go
new file mode 100644
index 000000000..f0056fa4c
--- /dev/null
+++ b/internal/hostactors/censuskey.go
@@ -0,0 +1,59 @@
+package hostactors
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "encoding/json"
+ "math"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// CensusKeyBinding is approved by the root enrollment coordinator after its
+// authenticated box session supplies a proof of possession. It is not an
+// unauthenticated enrollment request or a console-generated box report.
+type CensusKeyBinding struct {
+ Bucket string `json:"bucket"`
+ Box string `json:"box"`
+ Enrollment string `json:"enrollment"`
+ PublicKey ed25519.PublicKey `json:"public_key"`
+}
+
+func (d Directory) BindCensusKey(ctx context.Context, signer release.Signer, before Observed, binding CensusKeyBinding, proof []byte) (Observed, error) {
+ b, ok := before.Roster.Boxes[binding.Box]
+ if !ok || binding.Bucket != d.Bucket || binding.Enrollment != b.Enrollment || len(binding.PublicKey) != ed25519.PublicKeySize || before.ETag == "" || before.Roster.Revision == math.MaxUint64 {
+ return Observed{}, ErrRoster
+ }
+ raw, err := json.Marshal(binding)
+ if err != nil {
+ return Observed{}, err
+ }
+ if !ed25519.Verify(binding.PublicKey, raw, proof) {
+ return Observed{}, ErrRoster
+ }
+ if len(b.CensusPublicKey) != 0 {
+ if bytes.Equal(b.CensusPublicKey, binding.PublicKey) {
+ return before, nil
+ }
+ return Observed{}, ErrRoster
+ }
+ // One write-once extension to an existing enrollment. Ordinary Replace
+ // continues to reject changes to this field, including deletion/rotation.
+ next := Clone(before.Roster)
+ b.CensusPublicKey = append(ed25519.PublicKey(nil), binding.PublicKey...)
+ next.Boxes[binding.Box] = b
+ next.Revision++
+ if err = next.Validate(d.Bucket); err != nil {
+ return Observed{}, err
+ }
+ wire, err := release.SignRootEnvelope(signer, next)
+ if err != nil {
+ return Observed{}, err
+ }
+ tag, err := d.Store.Put(ctx, Key, wire, before.ETag)
+ if err != nil {
+ return Observed{}, err
+ }
+ return Observed{Roster: next, ETag: tag}, nil
+}
diff --git a/internal/hostactors/roster.go b/internal/hostactors/roster.go
index 7ef67ddb5..42bd34b19 100644
--- a/internal/hostactors/roster.go
+++ b/internal/hostactors/roster.go
@@ -23,17 +23,18 @@ var token = regexp.MustCompile(`^[0-9a-f]{32}$`)
type Cloud = boxid.Cloud
type Box struct {
- Enrollment string `json:"enrollment"`
- Namespace string `json:"namespace"`
- BoxID string `json:"box_id"`
- Label string `json:"label"`
- Region string `json:"region"`
- Pool string `json:"pool"`
- Cloud Cloud `json:"cloud"`
- EvidenceSHA256 string `json:"evidence_sha256"`
- State string `json:"state"`
- QuarantineRevision uint64 `json:"quarantine_revision"`
- LossIDs []string `json:"loss_ids"`
+ CensusPublicKey ed25519.PublicKey `json:"census_public_key,omitempty"`
+ Enrollment string `json:"enrollment"`
+ Namespace string `json:"namespace"`
+ BoxID string `json:"box_id"`
+ Label string `json:"label"`
+ Region string `json:"region"`
+ Pool string `json:"pool"`
+ Cloud Cloud `json:"cloud"`
+ EvidenceSHA256 string `json:"evidence_sha256"`
+ State string `json:"state"`
+ QuarantineRevision uint64 `json:"quarantine_revision"`
+ LossIDs []string `json:"loss_ids"`
}
type Console struct {
Compute *ComputeBinding `json:"compute,omitempty"`
@@ -94,6 +95,9 @@ func (r Roster) Validate(bucket string) error {
ids := map[string]bool{}
clouds := map[Cloud]bool{}
for key, b := range r.Boxes {
+ if len(b.CensusPublicKey) != 0 && len(b.CensusPublicKey) != ed25519.PublicKeySize {
+ return ErrRoster
+ }
if !segment(key) || b.BoxID != key || !token.MatchString(b.Enrollment) || ids[b.Enrollment] || b.Namespace == "" || b.Label == "" || b.Region == "" || b.Pool == "" || b.Cloud.Provider != "aws" || b.Cloud.Account == "" || b.Cloud.Region == "" || b.Cloud.InstanceID == "" || len(b.EvidenceSHA256) != 64 || clouds[b.Cloud] {
return ErrRoster
}
diff --git a/internal/hostclaim/adopt.go b/internal/hostclaim/adopt.go
index d4492f7a6..bc7ff4cf5 100644
--- a/internal/hostclaim/adopt.go
+++ b/internal/hostclaim/adopt.go
@@ -16,7 +16,7 @@ func Adopt(nh NewHolder, servingBox string, otherRegions, otherBoxes []string, o
if err != nil {
return err
}
- c.Holder = &Holder{Family: nh.Family, Key: nh.Key, Account: nh.Account, App: nh.App, Name: nh.Name,
+ c.Holder = &Holder{Acquisition: nh.Acquisition, Family: nh.Family, Key: nh.Key, Account: nh.Account, App: nh.App, Name: nh.Name,
Incarnation: inc, CreatedAt: nh.CreatedAt, Location: "instances/" + nh.Host, Stamp: c.Gen + 1, WriterBox: servingBox}
c.Op = Op{Kind: KindServing, Exec: servingBox, Participants: []string{servingBox}}
c.Touched = Touched{Complete: true, Regions: []string{nh.Region}, Boxes: []string{servingBox}}
diff --git a/internal/hostclaim/claim.go b/internal/hostclaim/claim.go
index 698444fe1..23d178589 100644
--- a/internal/hostclaim/claim.go
+++ b/internal/hostclaim/claim.go
@@ -37,7 +37,7 @@ const (
var knownKinds = map[string]bool{KindServing: true, KindCreating: true, KindDeleting: true, KindParking: true,
KindPurging: true, KindPlacing: true, KindMoving: true, KindLanding: true, KindReverting: true,
- KindRestoring: true, KindPushing: true}
+ KindRestoring: true, KindPushing: true, KindRelocating: true}
var (
ErrMalformed = errors.New("hostclaim: malformed claim")
@@ -45,6 +45,7 @@ var (
)
type Holder struct {
+ Acquisition string `json:"acq,omitempty"`
Family string `json:"family"` // "instance" | "sidecar"
Key string `json:"key"` // exact record key
Account string `json:"account"`
@@ -221,6 +222,9 @@ func (c *Claim) Validate() error {
if c == nil || c.V != schemaVersion || !controlname.ValidHostShape(c.Host) || c.Gen == 0 {
return fmt.Errorf("%w: header", ErrMalformed)
}
+ if err := validateRelocating(c); err != nil {
+ return err
+ }
if c.Op.Kind != "" && !knownKinds[c.Op.Kind] {
return fmt.Errorf("%w: unknown op kind %q", ErrMalformed, c.Op.Kind)
}
diff --git a/internal/hostclaim/create.go b/internal/hostclaim/create.go
index 4450413ba..c8fe97cc2 100644
--- a/internal/hostclaim/create.go
+++ b/internal/hostclaim/create.go
@@ -10,6 +10,7 @@ import (
var ErrRetiredIncarnation = errors.New("hostclaim: incarnation was retired on this host")
type NewHolder struct {
+ Acquisition string
Family, Key, Account, App, Name, Host string
RecordRegion string // Region encoded in the record; empty means home region.
Region string // Resolved storage region; must be non-empty.
@@ -70,7 +71,7 @@ func BeginCreate(nh NewHolder, recordBody []byte, opID, by string, now time.Time
return ErrRetiredIncarnation
}
}
- c.Holder = &Holder{Family: nh.Family, Key: nh.Key, Account: nh.Account, App: nh.App, Name: nh.Name,
+ c.Holder = &Holder{Acquisition: nh.Acquisition, Family: nh.Family, Key: nh.Key, Account: nh.Account, App: nh.App, Name: nh.Name,
Incarnation: inc, CreatedAt: nh.CreatedAt, Location: "instances/" + nh.Host + "/" + inc}
c.Op = Op{Kind: KindCreating, ID: opID, By: by, StartedAt: now.UTC(),
Intent: map[string]string{"key": nh.Key, "record": string(recordBody)}}
diff --git a/internal/hostclaim/fence.go b/internal/hostclaim/fence.go
new file mode 100644
index 000000000..bb90dd9bf
--- /dev/null
+++ b/internal/hostclaim/fence.go
@@ -0,0 +1,109 @@
+package hostclaim
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "strings"
+
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/termfence"
+)
+
+var ErrFence = errors.New("hostclaim: invalid termination fence")
+
+type FenceRef struct {
+ Enrollment string `json:"enrollment"`
+ Key string `json:"key"`
+ SHA256 string `json:"sha256"`
+}
+
+type FenceBinding struct {
+ Region string
+ RosterRevision uint64
+ RosterETag string
+}
+
+// FenceAuthority checks and identifies one authenticated roster observation.
+// Implementations must bind both the target and reporting worker enrollments,
+// cloud account, cloud region and terminal instance identity to that roster.
+type FenceAuthority interface {
+ ValidateFence(context.Context, termfence.Record) (FenceBinding, error)
+}
+
+type VerifiedFence struct {
+ ref FenceRef
+ record termfence.Record
+ binding FenceBinding
+}
+
+func validFenceRef(ref FenceRef) bool {
+ if len(ref.Enrollment) != 32 || ref.Enrollment != strings.ToLower(ref.Enrollment) {
+ return false
+ }
+ if _, err := hex.DecodeString(ref.Enrollment); err != nil {
+ return false
+ }
+ return ref.Key == "control/box-fences/"+ref.Enrollment+".json" && isHex64(ref.SHA256) && ref.SHA256 == strings.ToLower(ref.SHA256)
+}
+
+// VerifyFence re-reads the immutable worker-owned object, including on replay.
+// A previously verified reference alone never substitutes for verification.
+func VerifyFence(ctx context.Context, s release.Store, authority FenceAuthority, ref FenceRef) (VerifiedFence, error) {
+ if s == nil || authority == nil || !validFenceRef(ref) {
+ return VerifiedFence{}, ErrFence
+ }
+ raw, etag, err := s.Get(ctx, ref.Key)
+ if err != nil {
+ return VerifiedFence{}, err
+ }
+ sum := sha256.Sum256(raw)
+ if etag == "" || hex.EncodeToString(sum[:]) != ref.SHA256 {
+ return VerifiedFence{}, ErrFence
+ }
+ var f termfence.Record
+ if json.Unmarshal(raw, &f) != nil || f.Enrollment != ref.Enrollment || f.Box == "" || f.State != "terminated" || f.V != 1 || f.WrittenBy.Box == f.Box {
+ return VerifiedFence{}, ErrFence
+ }
+ binding, err := authority.ValidateFence(ctx, f)
+ if err != nil {
+ return VerifiedFence{}, err
+ }
+ if binding.Region == "" || binding.RosterRevision == 0 || binding.RosterETag == "" {
+ return VerifiedFence{}, ErrFence
+ }
+ return VerifiedFence{ref: ref, record: f, binding: binding}, nil
+}
+
+// VerifyOperationFence binds the journal key as well as the immutable record.
+// Return and recovery paths use this form rather than trusting a decoded ref.
+func VerifyOperationFence(ctx context.Context, s release.Store, authority FenceAuthority, op Op, box string) error {
+ if !validBoxReference(box) || op.ID == "" {
+ return ErrFence
+ }
+ var ref FenceRef
+ if json.Unmarshal([]byte(op.Journal["fence."+box]), &ref) != nil {
+ return ErrFence
+ }
+ verified, err := VerifyFence(ctx, s, authority, ref)
+ if err != nil {
+ return err
+ }
+ if verified.record.Box != box {
+ return ErrFence
+ }
+ return nil
+}
+
+func VerifyFenceForBox(ctx context.Context, s release.Store, authority FenceAuthority, ref FenceRef, box string) error {
+ verified, err := VerifyFence(ctx, s, authority, ref)
+ if err != nil {
+ return err
+ }
+ if !validBoxReference(box) || verified.record.Box != box {
+ return ErrFence
+ }
+ return nil
+}
diff --git a/internal/hostclaim/relocating.go b/internal/hostclaim/relocating.go
new file mode 100644
index 000000000..b5565460c
--- /dev/null
+++ b/internal/hostclaim/relocating.go
@@ -0,0 +1,142 @@
+package hostclaim
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "strconv"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+const KindRelocating = "relocating"
+
+// PreFlipGuard verifies the durable pre-flip conversion decision in the
+// authoritative instance record. That decision's record CAS competes with
+// MoveFlip and makes the move terminal for every cutting/flip transition.
+// It preserves the old move journal for replay, and binds this op, incarnation,
+// source and exact fence reference. A snapshot of an undecided record is not
+// an implementation of this interface.
+type PreFlipGuard interface {
+ CheckMovingBeforeFlip(context.Context, Observed, string, FenceRef) (string, error)
+}
+
+// RelocationRecoveryOwner reports replay ownership, never execution authority.
+func RelocationRecoveryOwner(c *Claim) (string, error) {
+ if c == nil || c.Op.Kind != KindRelocating || c.Op.Intent["region"] == "" {
+ return "", ErrWrongState
+ }
+ return "region:" + c.Op.Intent["region"], nil
+}
+
+func validateRelocating(c *Claim) error {
+ if c.Op.Kind != KindRelocating {
+ return nil
+ }
+ if c.Holder == nil || c.Op.Intent["region"] == "" || !validBoxReference(c.Op.Intent["source_box"]) || c.Op.Intent["source_box"] == "none" {
+ return ErrMalformed
+ }
+ switch c.Op.Phase {
+ case "starting", "releasing":
+ if c.Op.Exec != c.Op.Intent["source_box"] && c.Op.Exec != "none" {
+ return ErrMalformed
+ }
+ case "pinning":
+ if c.Op.Exec != "none" {
+ return ErrMalformed
+ }
+ case "installing", "installed", "probing", "probe-failed", "serving":
+ if !validBoxReference(c.Op.Exec) || c.Op.Exec == "none" || c.Op.Exec != c.Op.Intent["target_box"] {
+ return ErrMalformed
+ }
+ default:
+ return ErrMalformed
+ }
+ return nil
+}
+
+// ConvertMovingToRelocating performs exactly one claim CAS. It deliberately
+// exposes no transition capable of granting relocation execution. Callers must
+// be claim-enabled and first persist the record conversion decision.
+func ConvertMovingToRelocating(ctx context.Context, s release.Store, observed Observed, opID string, ref FenceRef, authority FenceAuthority, guard PreFlipGuard) (Observed, error) {
+ if guard == nil || observed.Absent || observed.Claim == nil || observed.ETag == "" {
+ return Observed{}, ErrWrongState
+ }
+ before := observed.Claim
+ if opID == "" || before.Op.ID != opID {
+ return Observed{}, ErrStaleOp
+ }
+ if before.Holder == nil || before.Op.Intent["admission_id"] == "" || before.Holder.Family != "instance" || before.Op.Kind != KindMoving || before.Op.Phase == "landing" || before.Op.Phase == "committed" {
+ return Observed{}, ErrWrongState
+ }
+ // Check the instance record even while the claim still says moving: the
+ // instance flip is earlier than final commit and can precede claim landing.
+ region, err := guard.CheckMovingBeforeFlip(ctx, observed, opID, ref)
+ if err != nil {
+ return Observed{}, err
+ }
+ fence, err := VerifyFence(ctx, s, authority, ref)
+ if err != nil {
+ return Observed{}, err
+ }
+ if region == "" || region != fence.binding.Region || before.Op.Intent["source_box"] != fence.record.Box || before.Op.Intent["source_enrollment"] != ref.Enrollment || before.Op.Intent["source_region"] != region || (before.Op.Exec != fence.record.Box && before.Op.Exec != "none" && before.Op.Exec != "") {
+ return Observed{}, ErrFence
+ }
+ raw, err := json.Marshal(ref)
+ if err != nil {
+ return Observed{}, err
+ }
+ return Update(ctx, s, observed, func(c *Claim) error {
+ if c.Op.ID != opID || c.Op.Kind != KindMoving || c.Op.Phase != before.Op.Phase {
+ return ErrStaleOp
+ }
+ key := "fence." + fence.record.Box
+ if old, ok := c.Op.Journal[key]; ok && old != string(raw) {
+ return fmt.Errorf("%w: fence reference already bound", ErrFence)
+ }
+ // Recheck the durable record decision immediately before the claim CAS.
+ // Its terminal state prevents any intervening flip.
+ currentRegion, err := guard.CheckMovingBeforeFlip(ctx, observed, opID, ref)
+ if err != nil {
+ return err
+ }
+ if currentRegion != region {
+ return ErrWrongState
+ }
+ if c.Op.Journal == nil {
+ c.Op.Journal = map[string]string{}
+ }
+ c.Op.Journal[key] = string(raw)
+ c.Op.Journal["reason"] = "source-terminated-before-flip"
+ c.Op.Journal["source_stamp"] = strconv.FormatUint(c.Holder.Stamp, 10)
+ c.Op.Journal["source_generation"] = c.Holder.Generation
+ c.Op.Journal["roster_revision"] = strconv.FormatUint(fence.binding.RosterRevision, 10)
+ c.Op.Journal["roster_etag"] = fence.binding.RosterETag
+ c.Op.Journal["admission_id"] = before.Op.Intent["admission_id"]
+ c.Op.Journal["conversion_claim_gen"] = strconv.FormatUint(before.Gen, 10)
+ c.Op.Journal["conversion_claim_etag"] = observed.ETag
+ c.Op.Kind, c.Op.Phase, c.Op.Exec = KindRelocating, "pinning", "none"
+ if c.Op.Intent == nil {
+ c.Op.Intent = map[string]string{}
+ }
+ c.Op.Intent["region"] = region
+ // The original move target is in a different region. It is not a selected
+ // same-region replacement. Keep the move history without authorizing it.
+ if target := c.Op.Intent["target_box"]; target != "" {
+ c.Op.Intent["move_target_box"] = target
+ delete(c.Op.Intent, "target_box")
+ }
+ if enrollment := c.Op.Intent["target_enrollment"]; enrollment != "" {
+ c.Op.Intent["move_target_enrollment"] = enrollment
+ delete(c.Op.Intent, "target_enrollment")
+ }
+ participants := make([]string, 0, len(c.Op.Participants))
+ for _, box := range c.Op.Participants {
+ if box != fence.record.Box {
+ participants = append(participants, box)
+ }
+ }
+ c.Op.Participants = participants
+ return nil
+ })
+}
diff --git a/internal/hostclaim/relocating_test.go b/internal/hostclaim/relocating_test.go
new file mode 100644
index 000000000..88d2cf2a7
--- /dev/null
+++ b/internal/hostclaim/relocating_test.go
@@ -0,0 +1,251 @@
+package hostclaim
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/boxid"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/termfence"
+)
+
+type c7Authority struct {
+ want termfence.Record
+ err error
+}
+
+func (a c7Authority) ValidateFence(_ context.Context, f termfence.Record) (FenceBinding, error) {
+ if a.err != nil {
+ return FenceBinding{}, a.err
+ }
+ if f != a.want {
+ return FenceBinding{}, ErrFence
+ }
+ return FenceBinding{Region: "ie", RosterRevision: 7, RosterETag: "roster-etag"}, nil
+}
+
+type c7Guard struct {
+ calls int
+ rejectAt int
+}
+
+func (g *c7Guard) CheckMovingBeforeFlip(_ context.Context, o Observed, id string, ref FenceRef) (string, error) {
+ g.calls++
+ if id != o.Claim.Op.ID || ref.Enrollment != o.Claim.Op.Intent["source_enrollment"] {
+ return "", ErrFence
+ }
+ if g.rejectAt > 0 && g.calls >= g.rejectAt {
+ return "", ErrWrongState
+ }
+ return "ie", nil
+}
+
+type c7Store struct {
+ release.Store
+ puts int
+}
+
+func (s *c7Store) Put(ctx context.Context, key string, body []byte, match string) (string, error) {
+ s.puts++
+ return s.Store.Put(ctx, key, body, match)
+}
+func c7Fixture(t *testing.T) (*c7Store, Observed, FenceRef, c7Authority) {
+ t.Helper()
+ ctx := context.Background()
+ s := &c7Store{Store: release.NewDirStore(t.TempDir())}
+ o := servingClaim(t, ctx, s)
+ var err error
+ o, err = Update(ctx, s, o, func(c *Claim) error {
+ c.Holder.Generation, c.Holder.Stamp = "gen-current", 17
+ c.Op = Op{ID: "move-1", Kind: KindMoving, Phase: "releasing", Exec: "adi-knitting",
+ Intent: map[string]string{"admission_id": "admission-existing", "source_box": "adi-knitting", "source_enrollment": strings.Repeat("a", 32), "source_region": "ie", "target_box": "bri-knitting", "target_enrollment": strings.Repeat("c", 32)},
+ Journal: map[string]string{"pin": "keep"}, Participants: []string{"adi-knitting", "bri-knitting"}}
+ c.Obligations = []Obligation{{ID: "staging", Op: "move-1", Kind: "park-replica", Region: "uk", Owner: "region:uk", Retirement: "retire-1", Park: &ParkScope{Location: "moves/" + testHost + "/move-1", Incarnation: c.Holder.Incarnation, EvidenceAt: time.Now().UTC(), RequestLimit: 120 * time.Second}}}
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ f := termfence.Record{V: 1, Box: "adi-knitting", Enrollment: strings.Repeat("a", 32), Cloud: boxid.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-source"}, State: "terminated", ObservedAt: "2026-10-06T00:00:00Z", Via: "describe", RequestID: "request", WrittenBy: termfence.WrittenBy{Box: "worker", Enrollment: strings.Repeat("b", 32)}}
+ raw, err := json.Marshal(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ digest := sha256.Sum256(raw)
+ ref := FenceRef{Enrollment: f.Enrollment, Key: "control/box-fences/" + f.Enrollment + ".json", SHA256: hex.EncodeToString(digest[:])}
+ if _, err := s.Put(ctx, ref.Key, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ s.puts = 0
+ return s, o, ref, c7Authority{want: f}
+}
+
+func TestC7ConversionPreservesOwnershipAndChores(t *testing.T) {
+ s, before, ref, a := c7Fixture(t)
+ guard := &c7Guard{}
+ after, err := ConvertMovingToRelocating(context.Background(), s, before, "move-1", ref, a, guard)
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := after.Claim
+ if c.Op.Journal["admission_id"] != "admission-existing" || c.Op.Journal["source_stamp"] != "17" || c.Op.Journal["source_generation"] != "gen-current" || c.Op.Journal["roster_revision"] != "7" || c.Op.Journal["roster_etag"] != "roster-etag" || c.Op.Journal["reason"] != "source-terminated-before-flip" || c.Op.Journal["conversion_claim_etag"] != before.ETag {
+ t.Fatalf("missing conversion authority: %+v", c.Op.Journal)
+ }
+ if s.puts != 1 || c.Gen != before.Claim.Gen+1 || guard.calls != 2 {
+ t.Fatalf("writes=%d gen=%d checks=%d", s.puts, c.Gen, guard.calls)
+ }
+ if !reflect.DeepEqual(c.Holder, before.Claim.Holder) || !reflect.DeepEqual(c.Obligations, before.Claim.Obligations) {
+ t.Fatal("holder or staging retirement changed")
+ }
+ if c.Op.ID != "move-1" || c.Op.Kind != KindRelocating || c.Op.Phase != "pinning" || c.Op.Exec != "none" || c.Op.Intent["region"] != "ie" {
+ t.Fatalf("wrong conversion: %+v", c.Op)
+ }
+ if c.Op.Intent["target_box"] != "" || c.Op.Intent["move_target_box"] != "bri-knitting" || !reflect.DeepEqual(c.Op.Participants, []string{"bri-knitting"}) {
+ t.Fatalf("target or barrier: %+v", c.Op)
+ }
+ var saved FenceRef
+ if err := json.Unmarshal([]byte(c.Op.Journal["fence.adi-knitting"]), &saved); err != nil || saved != ref {
+ t.Fatalf("journal reference: %+v %v", saved, err)
+ }
+ if owner, err := RelocationRecoveryOwner(c); err != nil || owner != "region:ie" {
+ t.Fatalf("owner=%q error=%v", owner, err)
+ }
+ if _, err := ConvertMovingToRelocating(context.Background(), s, before, "move-1", ref, a, &c7Guard{}); !errors.Is(err, ErrConflict) {
+ t.Fatalf("stale replay: %v", err)
+ }
+}
+
+func TestC7ConversionRefusesWithoutWriting(t *testing.T) {
+ for _, tc := range []string{"landing-record", "flip-during-verify", "landing-claim", "wrong-op", "wrong-source", "wrong-enrollment", "wrong-region", "foreign-exec", "journal-rebind", "authority", "digest", "no-guard", "missing-admission"} {
+ t.Run(tc, func(t *testing.T) {
+ s, o, ref, a := c7Fixture(t)
+ id := "move-1"
+ var guard PreFlipGuard = &c7Guard{}
+ switch tc {
+ case "landing-record":
+ guard = &c7Guard{rejectAt: 1}
+ case "flip-during-verify":
+ guard = &c7Guard{rejectAt: 2}
+ case "landing-claim":
+ o.Claim.Op.Kind = KindLanding
+ case "wrong-op":
+ id = "other"
+ case "wrong-source":
+ o.Claim.Op.Intent["source_box"] = "other"
+ case "wrong-enrollment":
+ o.Claim.Op.Intent["source_enrollment"] = strings.Repeat("d", 32)
+ case "wrong-region":
+ o.Claim.Op.Intent["source_region"] = "uk"
+ case "foreign-exec":
+ o.Claim.Op.Exec = "target"
+ case "journal-rebind":
+ o.Claim.Op.Journal["fence.adi-knitting"] = `{"enrollment":"other"}`
+ case "authority":
+ a.err = ErrFence
+ case "digest":
+ ref.SHA256 = strings.Repeat("0", 64)
+ case "no-guard":
+ guard = nil
+ case "missing-admission":
+ delete(o.Claim.Op.Intent, "admission_id")
+ }
+ raw, _ := json.Marshal(o.Claim)
+ if _, err := ConvertMovingToRelocating(context.Background(), s, o, id, ref, a, guard); err == nil {
+ t.Fatal("accepted")
+ }
+ unchanged, _ := json.Marshal(o.Claim)
+ if s.puts != 0 || string(raw) != string(unchanged) {
+ t.Fatal("refusal mutated claim")
+ }
+ })
+ }
+}
+
+func TestC7VerifyFenceReReadsExactObject(t *testing.T) {
+ ctx := context.Background()
+ s, _, ref, a := c7Fixture(t)
+ if _, err := VerifyFence(ctx, s, a, ref); err != nil {
+ t.Fatal(err)
+ }
+ for _, tc := range []string{"bad-key", "bad-enrollment", "bad-digest", "missing", "changed-bytes", "authority"} {
+ t.Run(tc, func(t *testing.T) {
+ s, _, ref, a := c7Fixture(t)
+ switch tc {
+ case "bad-key":
+ ref.Key = "control/box-fences/other.json"
+ case "bad-enrollment":
+ ref.Enrollment = "../escape"
+ case "bad-digest":
+ ref.SHA256 = strings.Repeat("f", 64)
+ case "missing":
+ if err := s.Delete(ctx, ref.Key); err != nil {
+ t.Fatal(err)
+ }
+ case "changed-bytes":
+ raw, etag, err := s.Get(ctx, ref.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = s.Put(ctx, ref.Key, append(raw, ' '), etag); err != nil {
+ t.Fatal(err)
+ }
+ case "authority":
+ a.err = ErrFence
+ }
+ if _, err := VerifyFence(ctx, s, a, ref); err == nil {
+ t.Fatal("accepted")
+ }
+ })
+ }
+}
+
+func TestC7RelocatingPhaseValidation(t *testing.T) {
+ for _, phase := range []string{"starting", "releasing", "pinning", "installing", "installed", "probing", "probe-failed", "serving", "unknown"} {
+ t.Run(phase, func(t *testing.T) {
+ _, o, _, _ := c7Fixture(t)
+ c := o.Claim
+ c.Op.Kind, c.Op.Phase = KindRelocating, phase
+ c.Op.Intent["region"] = "ie"
+ switch phase {
+ case "starting", "releasing":
+ c.Op.Exec = "adi-knitting"
+ case "pinning":
+ c.Op.Exec = "none"
+ default:
+ c.Op.Exec = "bri-knitting"
+ }
+ if err := c.Validate(); (err != nil) != (phase == "unknown") {
+ t.Fatalf("phase %s: %v", phase, err)
+ }
+ if phase == "pinning" {
+ c.Op.Exec = "bri-knitting"
+ if c.Validate() == nil {
+ t.Fatal("pinning granted execution")
+ }
+ }
+ })
+ }
+}
+
+func TestC7JournalFenceCannotNameAnotherBox(t *testing.T) {
+ s, o, ref, a := c7Fixture(t)
+ raw, err := json.Marshal(ref)
+ if err != nil {
+ t.Fatal(err)
+ }
+ o.Claim.Op.Journal["fence.adi-knitting"] = string(raw)
+ if err := VerifyOperationFence(context.Background(), s, a, o.Claim.Op, "adi-knitting"); err != nil {
+ t.Fatal(err)
+ }
+ o.Claim.Op.Journal["fence.other"] = string(raw)
+ if err := VerifyOperationFence(context.Background(), s, a, o.Claim.Op, "other"); err == nil {
+ t.Fatal("journal key rebound to another box")
+ }
+}
diff --git a/internal/hostfence/claim_fence.go b/internal/hostfence/claim_fence.go
new file mode 100644
index 000000000..e4e09e6f6
--- /dev/null
+++ b/internal/hostfence/claim_fence.go
@@ -0,0 +1,38 @@
+package hostfence
+
+import (
+ "context"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/termfence"
+)
+
+// ClaimFenceAuthority authenticates the roster without creating the reverse
+// hostclaim -> hostactors dependency. Immutable evidence remains valid after
+// its reporter retires; enrollment/cloud binding must still be retained.
+type ClaimFenceAuthority struct {
+ Directory hostactors.Directory
+ Now func() time.Time
+}
+
+var _ hostclaim.FenceAuthority = ClaimFenceAuthority{}
+
+func (a ClaimFenceAuthority) ValidateFence(ctx context.Context, f termfence.Record) (hostclaim.FenceBinding, error) {
+ if a.Now == nil || f.WrittenBy.Box == f.Box {
+ return hostclaim.FenceBinding{}, hostclaim.ErrFence
+ }
+ o, err := a.Directory.Read(ctx)
+ if err != nil {
+ return hostclaim.FenceBinding{}, err
+ }
+ if err := hostactors.ValidateTerminationFence(o, f.Box, f, a.Now()); err != nil {
+ return hostclaim.FenceBinding{}, err
+ }
+ return hostclaim.FenceBinding{Region: o.Roster.Boxes[f.Box].Region, RosterRevision: o.Roster.Revision, RosterETag: o.ETag}, nil
+}
+
+func ClaimFenceRef(ref EvidenceRef) hostclaim.FenceRef {
+ return hostclaim.FenceRef{Enrollment: ref.Enrollment, Key: ref.Key, SHA256: ref.SHA256}
+}
diff --git a/internal/hostfence/claim_fence_test.go b/internal/hostfence/claim_fence_test.go
new file mode 100644
index 000000000..6ccaa8ae0
--- /dev/null
+++ b/internal/hostfence/claim_fence_test.go
@@ -0,0 +1,72 @@
+package hostfence
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "testing"
+
+ "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+ "github.com/carlosframework/platform/internal/awsinv"
+ "github.com/carlosframework/platform/internal/hostclaim"
+)
+
+func TestC7FenceReaderAuthenticatesRosterAndCloud(t *testing.T) {
+ for _, tc := range []string{"valid", "enrollment", "account", "cloud-region", "instance", "reporter", "nonterminal", "signature", "self-reporter"} {
+ t.Run(tc, func(t *testing.T) {
+ ctx := context.Background()
+ w, s := terminationFixture(t)
+ w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) {
+ return terminatedEC2{state: types.InstanceStateNameTerminated, owner: "123"}, nil
+ }
+ got, err := w.Describe(ctx, "dead")
+ if err != nil {
+ t.Fatal(err)
+ }
+ ref := ClaimFenceRef(got)
+ raw, etag, err := s.Get(ctx, ref.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var f Termination
+ if err = json.Unmarshal(raw, &f); err != nil {
+ t.Fatal(err)
+ }
+ switch tc {
+ case "enrollment":
+ f.Enrollment = "cccccccccccccccccccccccccccccccc"
+ case "account":
+ f.Cloud.Account = "456"
+ case "cloud-region":
+ f.Cloud.Region = "eu-west-2"
+ case "instance":
+ f.Cloud.InstanceID = "i-other"
+ case "reporter":
+ f.WrittenBy.Enrollment = "cccccccccccccccccccccccccccccccc"
+ case "nonterminal":
+ f.State = "shutting-down"
+ case "self-reporter":
+ f.WrittenBy.Box = f.Box
+ f.WrittenBy.Enrollment = f.Enrollment
+ case "signature":
+ w.Directory.PublicKey = make([]byte, 32)
+ }
+ raw, err = json.Marshal(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Deliberately replace the object in this local adversarial fixture and
+ // recompute its digest: a digest match alone must not grant authority.
+ if _, err = s.Put(ctx, ref.Key, raw, etag); err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(raw)
+ ref.SHA256 = hex.EncodeToString(sum[:])
+ _, err = hostclaim.VerifyFence(ctx, s, ClaimFenceAuthority{Directory: w.Directory, Now: w.Now}, ref)
+ if (err == nil) != (tc == "valid") {
+ t.Fatalf("%s: %v", tc, err)
+ }
+ })
+ }
+}
diff --git a/internal/hostops/backfill.go b/internal/hostops/backfill.go
new file mode 100644
index 000000000..e20b19823
--- /dev/null
+++ b/internal/hostops/backfill.go
@@ -0,0 +1,380 @@
+package hostops
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "os"
+ "sort"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/domains"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+const BackfillKey = "control/hosts-backfill.json"
+const HostsQuarantineKey = "control/hosts-quarantine.json"
+
+type CensusInstallation struct {
+ Placement string `json:"placement"`
+ Host string `json:"host"`
+ Account string `json:"account"`
+ App string `json:"app"`
+ Family string `json:"family"`
+ Key string `json:"key"`
+ Incarnation string `json:"incarnation"`
+ Stamp uint64 `json:"stamp"`
+ Local bool `json:"local"`
+ Route bool `json:"route"`
+}
+type CensusReport struct {
+ Box string `json:"box"`
+ Enrollment string `json:"enrollment"`
+ BootID string `json:"boot_id"`
+ Reservation string `json:"reservation"`
+ DirectoryRevision uint64 `json:"directory_revision"`
+ CapturedAt time.Time `json:"captured_at"`
+ Complete bool `json:"complete"`
+ Installations []CensusInstallation `json:"installations"`
+}
+type Census struct {
+ Records []Record
+ Reports map[string]CensusReport
+ Roster hostactors.Observed
+}
+type BackfillHost struct {
+ Key string `json:"key"`
+ ETag string `json:"etag"`
+ Acquisition string `json:"acq"`
+ Region string `json:"region"`
+ Box string `json:"box"`
+}
+type BackfillPlan struct {
+ V int `json:"v"`
+ ID string `json:"id"`
+ Phase string `json:"phase"`
+ DirectoryRevision uint64 `json:"directory_revision"`
+ CreatedAt time.Time `json:"created_at"`
+ Reports map[string]string `json:"reports"`
+ Hosts map[string]BackfillHost `json:"hosts"`
+}
+type HostsQuarantine struct {
+ V int `json:"v"`
+ Hosts map[string]string `json:"hosts"`
+}
+type CensusEnvelope struct {
+ Report CensusReport `json:"report"`
+ Signature []byte `json:"signature"`
+}
+
+type Backfill struct {
+ mu sync.Mutex
+ cached *Census
+ cachedTag string
+ Store release.Store
+ Directory hostactors.Directory
+}
+type ProductionAuthority struct {
+ Store release.Store
+ Directory hostactors.Directory
+ Now func() time.Time
+ ConsoleID, Enrollment, Namespace string
+ Signer release.Signer
+ Evidence hostactors.Evidence
+ Backfill *Backfill
+}
+
+func reportKey(reservation, box string) (string, error) {
+ if !safeSegment(reservation) || !safeSegment(box) {
+ return "", ErrIdentity
+ }
+ return "control/host-census/" + reservation + "/" + box + ".json", nil
+}
+func safeSegment(s string) bool {
+ if s == "" || s == "." || s == ".." {
+ return false
+ }
+ for _, c := range s {
+ if c == '/' || c == '\\' || c == 0 {
+ return false
+ }
+ }
+ return true
+}
+func CheckHostQuarantine(ctx context.Context, s release.Store, host string) error {
+ raw, _, err := s.Get(ctx, HostsQuarantineKey)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ var q HostsQuarantine
+ if decode(raw, &q) != nil || q.V != 1 || q.Hosts == nil {
+ return ErrIdentity
+ }
+ if _, ok := q.Hosts[host]; ok {
+ return ErrPending
+ }
+ return nil
+}
+func (b *Backfill) read(ctx context.Context) (BackfillPlan, string, error) {
+ p, tag, err := b.readPlan(ctx)
+ if err != nil {
+ return p, "", err
+ }
+ if p.Phase != "adopting" {
+ return p, "", ErrContract
+ }
+ return p, tag, nil
+}
+func (b *Backfill) readPlan(ctx context.Context) (BackfillPlan, string, error) {
+ var p BackfillPlan
+ raw, tag, err := b.Store.Get(ctx, BackfillKey)
+ if err != nil {
+ return p, "", err
+ }
+ if tag == "" || release.VerifyRootEnvelope(b.Directory.PublicKey, raw, &p) != nil || p.V != 1 || !safeSegment(p.ID) || p.CreatedAt.IsZero() || (p.Phase == "adopting" && (p.Hosts == nil || p.Reports == nil)) {
+ return p, "", ErrContract
+ }
+ return p, tag, nil
+}
+func (a ProductionAuthority) census(ctx context.Context, account, app, reservation string, expectedRevision uint64, digests map[string]string) (Census, error) {
+ out := Census{Reports: map[string]CensusReport{}}
+ roster, err := a.Directory.Read(ctx)
+ if err != nil {
+ return out, err
+ }
+ if expectedRevision != 0 && roster.Roster.Revision != expectedRevision {
+ return out, ErrPending
+ }
+ out.Roster = roster
+ apps, err := Apps(ctx, a.Store)
+ if err != nil {
+ return out, err
+ }
+ allRecords := map[string]Record{}
+ for _, p := range apps {
+ records, err := Records(ctx, a.Store, p.Account, p.Name)
+ if err != nil {
+ return out, err
+ }
+ for _, r := range records {
+ if _, ok := allRecords[r.Holder.Host]; ok {
+ return out, ErrIdentity
+ }
+ allRecords[r.Holder.Host] = r
+ if account == "" || (p.Account == account && p.Name == app) {
+ out.Records = append(out.Records, r)
+ }
+ }
+ }
+ for ref, box := range roster.Roster.Boxes {
+ // Retired membership is retained until its inventory is proven empty.
+ key, err := reportKey(reservation, ref)
+ if err != nil {
+ return out, err
+ }
+ raw, tag, err := a.Store.Get(ctx, key)
+ if err != nil {
+ return out, err
+ }
+ if digests != nil {
+ sum := sha256.Sum256(raw)
+ if digests[ref] != hex.EncodeToString(sum[:]) {
+ return out, ErrIdentity
+ }
+ }
+ var envelope CensusEnvelope
+ if decode(raw, &envelope) != nil {
+ return out, ErrIdentity
+ }
+ report := envelope.Report
+ canonical, e := json.Marshal(report)
+ if e != nil || len(box.CensusPublicKey) != ed25519.PublicKeySize || !ed25519.Verify(box.CensusPublicKey, canonical, envelope.Signature) {
+ return out, ErrIdentity
+ }
+ if tag == "" || !report.Complete || report.Box != ref || report.Enrollment != box.Enrollment || report.Reservation != reservation || report.DirectoryRevision != roster.Roster.Revision || report.BootID == "" || report.CapturedAt.IsZero() || report.Installations == nil {
+ return out, ErrIdentity
+ }
+ seen := map[string]bool{}
+ for _, i := range report.Installations {
+ if i.Host == "" || seen[i.Host] || (!i.Route && !i.Local) {
+ return out, ErrIdentity
+ }
+ seen[i.Host] = true
+ r, ok := allRecords[i.Host]
+ if !ok || i.Account != r.Holder.Account || i.App != r.Holder.App || i.Key != r.Holder.Key || i.Family != r.Holder.Family {
+ return out, ErrIdentity
+ }
+ inc, err := r.Incarnation()
+ if err != nil || (i.Incarnation != "" && i.Incarnation != inc) {
+ return out, ErrIdentity
+ }
+ }
+ out.Reports[ref] = report
+ }
+ if digests != nil && len(digests) != len(out.Reports) {
+ return out, ErrIdentity
+ }
+ fresh, err := a.Directory.Read(ctx)
+ if err != nil {
+ return out, err
+ }
+ if fresh.ETag != roster.ETag {
+ return out, ErrPending
+ }
+ return out, nil
+}
+func (a ProductionAuthority) Census(ctx context.Context, account, app, reservation string) (Census, error) {
+ return a.census(ctx, account, app, reservation, 0, nil)
+}
+func (a ProductionAuthority) Inventory(ctx context.Context, obs hostclaim.Observed) ([]string, []string, error) {
+ roster, err := a.Directory.Read(ctx)
+ if err != nil {
+ return nil, nil, err
+ }
+ if len(roster.Roster.Regions) == 0 {
+ return nil, nil, ErrContract
+ }
+ regions := make([]string, 0, len(roster.Roster.Regions))
+ boxes := make([]string, 0, len(roster.Roster.Boxes))
+ for r := range roster.Roster.Regions {
+ regions = append(regions, r)
+ }
+ for b := range roster.Roster.Boxes {
+ boxes = append(boxes, b)
+ }
+ sort.Strings(regions)
+ sort.Strings(boxes)
+ return regions, boxes, nil
+}
+func (a ProductionAuthority) Adoption(ctx context.Context, r Record) (region, box string, regions, boxes []string, err error) {
+ if a.Backfill == nil {
+ err = ErrContract
+ return
+ }
+ return a.Backfill.Adoption(ctx, r)
+}
+func (b *Backfill) Adoption(ctx context.Context, r Record) (region, box string, regions, boxes []string, err error) {
+ if err = CheckHostQuarantine(ctx, b.Store, r.Holder.Host); err != nil {
+ return
+ }
+ p, tag, e := b.read(ctx)
+ if e != nil {
+ err = e
+ return
+ }
+ selected, ok := p.Hosts[r.Holder.Host]
+ if !ok || selected.Key != r.Holder.Key || selected.ETag != r.ETag || selected.Acquisition == "" || selected.Region == "" || selected.Box == "" {
+ err = ErrIdentity
+ return
+ }
+ a := ProductionAuthority{Store: b.Store, Directory: b.Directory}
+ b.mu.Lock()
+ if b.cachedTag != tag {
+ b.cached = nil
+ }
+ var census Census
+ if b.cached != nil {
+ census = *b.cached
+ } else {
+ census, e = a.census(ctx, "", "", p.ID, p.DirectoryRevision, p.Reports)
+ if e == nil {
+ b.cached = &census
+ b.cachedTag = tag
+ }
+ }
+ b.mu.Unlock()
+ if e != nil {
+ err = e
+ return
+ }
+ current, e := a.Directory.Read(ctx)
+ if e != nil {
+ err = e
+ return
+ }
+ if current.ETag != census.Roster.ETag {
+ err = ErrPending
+ return
+ }
+ owners := map[string]bool{}
+ found := false
+ for _, rec := range census.Records {
+ if rec.Holder.Host == r.Holder.Host {
+ found = rec.ETag == r.ETag && rec.Holder.Key == r.Holder.Key
+ }
+ }
+ for ref, report := range census.Reports {
+ for _, i := range report.Installations {
+ if i.Host == r.Holder.Host {
+ owners[ref] = true
+ }
+ }
+ }
+ if !found || len(owners) != 1 || !owners[selected.Box] {
+ err = ErrIdentity
+ return
+ }
+ enrolled, ok := census.Roster.Roster.Boxes[selected.Box]
+ if !ok || enrolled.Region != selected.Region || enrolled.State != "active" {
+ err = ErrIdentity
+ return
+ }
+ if r.Holder.Family == "instance" {
+ var instance provision.Record
+ if json.Unmarshal(r.Body, &instance) != nil || len(instance.Replicas) != 0 || provision.ClaimEverywhere(instance) {
+ err = ErrIdentity
+ return
+ }
+ }
+ raw, _, e := b.Store.Get(ctx, domains.ClaimKey(r.Holder.Host))
+ if e != nil {
+ err = e
+ return
+ }
+ var ownership domains.Claim
+ inc, e := r.Incarnation()
+ if e != nil {
+ err = e
+ return
+ }
+ if decode(raw, &ownership) != nil || ownership.Host != r.Holder.Host || ownership.Account != r.Holder.Account || ownership.App != r.Holder.App || ownership.Acquisition != selected.Acquisition || ownership.RecordIncarnation != inc || ownership.OwnershipState != "committed" || ownership.Pending {
+ err = ErrIdentity
+ return
+ }
+ if r.Holder.Acquisition != selected.Acquisition {
+ err = ErrIdentity
+ return
+ }
+ _, freshTag, e := b.read(ctx)
+ if e != nil {
+ err = e
+ return
+ }
+ if freshTag != tag {
+ err = ErrPending
+ return
+ }
+ for region := range census.Roster.Roster.Regions {
+ if region != selected.Region {
+ regions = append(regions, region)
+ }
+ }
+ for ref := range census.Roster.Roster.Boxes {
+ if ref != selected.Box {
+ boxes = append(boxes, ref)
+ }
+ }
+ sort.Strings(regions)
+ sort.Strings(boxes)
+ return selected.Region, selected.Box, regions, boxes, nil
+}
diff --git a/internal/hostops/backfill_controller.go b/internal/hostops/backfill_controller.go
new file mode 100644
index 000000000..08ebd8567
--- /dev/null
+++ b/internal/hostops/backfill_controller.go
@@ -0,0 +1,123 @@
+package hostops
+
+import (
+ "context"
+ "errors"
+ "os"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// BackfillAuthority deliberately grants only the frozen backfill's adoption.
+// Mutation doors use ProductionAuthority once their full construction lands.
+type BackfillAuthority struct{ p ProductionAuthority }
+
+func (a BackfillAuthority) Inventory(ctx context.Context, obs hostclaim.Observed) ([]string, []string, error) {
+ return a.p.Inventory(ctx, obs)
+}
+func (a BackfillAuthority) Adoption(ctx context.Context, r Record) (string, string, []string, []string, error) {
+ return a.p.Adoption(ctx, r)
+}
+func (a BackfillAuthority) ClaimBackfill() *Backfill { return a.p.Backfill }
+
+func (a BackfillAuthority) Create(context.Context, hostclaim.Observed, Record, string) error {
+ return ErrContract
+}
+func (a BackfillAuthority) Executor(context.Context, hostclaim.Observed, Record) (string, error) {
+ return "", ErrContract
+}
+func (a BackfillAuthority) StartDelete(context.Context, hostclaim.Observed, Record) error {
+ return ErrContract
+}
+func (a BackfillAuthority) FirstPhase(context.Context, hostclaim.Observed) error { return ErrContract }
+func (a BackfillAuthority) PurgeRetirement(context.Context, hostclaim.Observed) error {
+ return ErrContract
+}
+func NewBackfillController(s release.Store, d hostactors.Directory, now func() time.Time, newID func() string) Controller {
+ b := &Backfill{Store: s, Directory: d}
+ return Controller{Store: s, Now: now, NewID: newID, Authority: BackfillAuthority{ProductionAuthority{Store: s, Directory: d, Backfill: b, Now: now}}}
+}
+
+func ReconcileClaims(ctx context.Context, c Controller, adopt func(context.Context, string) (hostclaim.Observed, error)) error {
+ if adopt == nil {
+ return ErrContract
+ }
+ var backfill *Backfill
+ if provider, ok := c.Authority.(interface{ ClaimBackfill() *Backfill }); ok {
+ backfill = provider.ClaimBackfill()
+ }
+ if backfill != nil {
+ plan, _, err := backfill.readPlan(ctx)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ if plan.Phase != "adopting" {
+ return nil
+ }
+ }
+ apps, err := Apps(ctx, c.Store)
+ if err != nil {
+ return err
+ }
+ var all []error
+ // Collect the entire listing before assigning any hostname. A broken page
+ // cannot authorize an adoption against only the prefix read so far.
+ var records []Record
+ seen := map[string]bool{}
+ for _, app := range apps {
+ rs, err := Records(ctx, c.Store, app.Account, app.Name)
+ if err != nil {
+ return err
+ }
+ for _, r := range rs {
+ if seen[r.Holder.Host] {
+ return ErrIdentity
+ }
+ seen[r.Holder.Host] = true
+ records = append(records, r)
+ }
+ }
+ for _, listed := range records {
+ r, err := ReadRecord(ctx, c.Store, listed.Holder.Key)
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ if err != nil {
+ all = append(all, err)
+ continue
+ }
+ if err = CheckHostQuarantine(ctx, c.Store, r.Holder.Host); err != nil {
+ all = append(all, err)
+ continue
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, r.Holder.Host)
+ if err != nil {
+ all = append(all, err)
+ continue
+ }
+ if !obs.Absent && (obs.Claim.Holder == nil || obs.Claim.Holder.Acquisition != "" || r.Holder.Acquisition == "") {
+ continue
+ }
+ if _, err = adopt(ctx, r.Holder.Key); err != nil {
+ all = append(all, err)
+ }
+ }
+ return errors.Join(all...)
+}
+func (c Controller) ReconcileClaims(ctx context.Context) error {
+ return ReconcileClaims(ctx, c, func(ctx context.Context, key string) (hostclaim.Observed, error) {
+ rec, err := ReadRecord(ctx, c.Store, key)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ return c.Adopt(ctx, rec)
+ })
+}
+
+func (a ProductionAuthority) ClaimBackfill() *Backfill { return a.Backfill }
diff --git a/internal/hostops/backfill_publish.go b/internal/hostops/backfill_publish.go
new file mode 100644
index 000000000..9f770662c
--- /dev/null
+++ b/internal/hostops/backfill_publish.go
@@ -0,0 +1,162 @@
+package hostops
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "github.com/carlosframework/platform/internal/hostactors"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// Publish freezes exact report bytes into #232's already-fenced backfill.
+// The collecting manifest is the global create/record-writer freeze. Its ETag
+// is supplied by the #232 coordinator, which retains that freeze through done.
+func (b *Backfill) Publish(ctx context.Context, signer release.Signer, collectingETag string) error {
+ raw, tag, err := b.Store.Get(ctx, BackfillKey)
+ if err != nil {
+ return err
+ }
+ var p BackfillPlan
+ if tag != collectingETag || tag == "" || release.VerifyRootEnvelope(b.Directory.PublicKey, raw, &p) != nil || p.V != 1 || p.Phase != "collecting" || p.ID == "" || p.CreatedAt.IsZero() || p.Hosts == nil {
+ return ErrContract
+ }
+ a := ProductionAuthority{Store: b.Store, Directory: b.Directory}
+ census, err := a.census(ctx, "", "", p.ID, p.DirectoryRevision, nil)
+ if err != nil {
+ return err
+ }
+ p.Reports = map[string]string{}
+ for box, report := range census.Reports {
+ if report.CapturedAt.Before(p.CreatedAt) {
+ return ErrIdentity
+ }
+ key, err := reportKey(p.ID, box)
+ if err != nil {
+ return err
+ }
+ body, _, err := b.Store.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ var envelope CensusEnvelope
+ if json.Unmarshal(body, &envelope) != nil || envelope.Report.Reservation != report.Reservation || !envelope.Report.Complete {
+ return ErrIdentity
+ }
+ sum := sha256.Sum256(body)
+ p.Reports[box] = hex.EncodeToString(sum[:])
+ }
+ for _, r := range census.Records {
+ if err := CheckHostQuarantine(ctx, b.Store, r.Holder.Host); err != nil {
+ return err
+ }
+ selected, ok := p.Hosts[r.Holder.Host]
+ if !ok || selected.Key != r.Holder.Key || selected.ETag != r.ETag || selected.Acquisition == "" || r.Holder.Acquisition != selected.Acquisition {
+ return ErrIdentity
+ }
+ }
+ if len(p.Hosts) != len(census.Records) {
+ return ErrIdentity
+ }
+ p.Phase = "adopting"
+ body, err := release.SignRootEnvelope(signer, p)
+ if err != nil {
+ return err
+ }
+ _, err = b.Store.Put(ctx, BackfillKey, body, tag)
+ return err
+}
+
+// PublishCensusReport is called by the guarded box census collector. Each box
+// owns its private signing key; the root-signed roster binds its public key to
+// verified enrollment. The console has only that public key.
+func PublishCensusReport(ctx context.Context, store release.Store, d hostactors.Directory, report CensusReport, private ed25519.PrivateKey) error {
+ if len(private) != ed25519.PrivateKeySize {
+ return ErrContract
+ }
+ roster, err := d.Read(ctx)
+ if err != nil {
+ return err
+ }
+ box, ok := roster.Roster.Boxes[report.Box]
+ if !ok || report.Enrollment != box.Enrollment || report.DirectoryRevision != roster.Roster.Revision || !bytes.Equal(private.Public().(ed25519.PublicKey), box.CensusPublicKey) || report.BootID == "" || !report.Complete || report.Installations == nil || report.CapturedAt.IsZero() {
+ return ErrIdentity
+ }
+ canonical, err := json.Marshal(report)
+ if err != nil {
+ return err
+ }
+ raw, err := json.Marshal(CensusEnvelope{Report: report, Signature: ed25519.Sign(private, canonical)})
+ if err != nil {
+ return err
+ }
+ key, err := reportKey(report.Reservation, report.Box)
+ if err != nil {
+ return err
+ }
+ if _, err = store.Put(ctx, key, raw, ""); err == nil {
+ return nil
+ }
+ if !errors.Is(err, release.ErrCASConflict) {
+ return err
+ }
+ previous, _, err := store.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ if !bytes.Equal(previous, raw) {
+ return ErrIdentity
+ }
+ return nil
+}
+
+// AuthorizeBackfillApp is the only legacy app-admission initialization proof.
+// It accepts the same complete frozen census as host adoption, including apps
+// with no host records. Empty results from a failed listing never qualify.
+func (a ProductionAuthority) AuthorizeBackfillApp(ctx context.Context, account, app string) (string, error) {
+ if a.Backfill == nil || !safeSegment(account) || !safeSegment(app) {
+ return "", ErrContract
+ }
+ p, tag, err := a.Backfill.read(ctx)
+ if err != nil {
+ return "", err
+ }
+ census, err := a.census(ctx, account, app, p.ID, p.DirectoryRevision, p.Reports)
+ if err != nil {
+ return "", err
+ }
+ apps, err := Apps(ctx, a.Store)
+ if err != nil {
+ return "", err
+ }
+ found := false
+ for _, candidate := range apps {
+ if candidate.Account == account && candidate.Name == app {
+ found = true
+ }
+ }
+ if !found {
+ return "", ErrIdentity
+ }
+ for _, r := range census.Records {
+ planned, ok := p.Hosts[r.Holder.Host]
+ if !ok || planned.Key != r.Holder.Key || planned.ETag != r.ETag || planned.Acquisition == "" || planned.Acquisition != r.Holder.Acquisition {
+ return "", ErrIdentity
+ }
+ if err = CheckHostQuarantine(ctx, a.Store, r.Holder.Host); err != nil {
+ return "", err
+ }
+ }
+ _, fresh, err := a.Backfill.read(ctx)
+ if err != nil {
+ return "", err
+ }
+ if fresh != tag {
+ return "", ErrPending
+ }
+ return p.ID, nil
+}
diff --git a/internal/hostops/backfill_test.go b/internal/hostops/backfill_test.go
new file mode 100644
index 000000000..496d55747
--- /dev/null
+++ b/internal/hostops/backfill_test.go
@@ -0,0 +1,357 @@
+package hostops
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/domains"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func TestReconcileFrozenBackfillAndQuarantine(t *testing.T) {
+ testReconcileFrozenBackfill(t, false, "")
+}
+
+func TestReconcileFrozenBackfillRepairsLegacyHolder(t *testing.T) {
+ testReconcileFrozenBackfill(t, true, "")
+}
+
+func testReconcileFrozenBackfill(t *testing.T, legacy bool, scenario string) {
+ t.Helper()
+ ctx := context.Background()
+ f := newFixture(t)
+ pub, key, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sg := release.RootSigner(key)
+ roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Regions: map[string]hostactors.Region{"ie": {}, "us": {}}, Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
+ boxKeys := map[string]ed25519.PrivateKey{}
+ for i, ref := range []string{"box-a", "box-b"} {
+ boxPub, boxKey, e := ed25519.GenerateKey(rand.Reader)
+ if e != nil {
+ t.Fatal(e)
+ }
+ boxKeys[ref] = boxKey
+ region := "ie"
+ en := strings.Repeat("a", 32)
+ if i == 1 {
+ region = "us"
+ en = strings.Repeat("b", 32)
+ }
+ roster.Boxes[ref] = hostactors.Box{CensusPublicKey: boxPub, Enrollment: en, Namespace: "platform", BoxID: ref, Label: ref, Region: region, Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: region, InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"}
+ }
+ if scenario == "two-box-region" || scenario == "two-owners" {
+ box := roster.Boxes["box-b"]
+ box.Region, box.Cloud.Region = "ie", "ie"
+ roster.Boxes["box-b"] = box
+ }
+ if scenario == "inactive-owner" || scenario == "wrong-region" {
+ box := roster.Boxes["box-a"]
+ if scenario == "inactive-owner" {
+ box.State = "retired"
+ } else {
+ box.Region, box.Cloud.Region = "us", "us"
+ }
+ roster.Boxes["box-a"] = box
+ }
+ put := func(k string, v any, signed bool) string {
+ t.Helper()
+ var raw []byte
+ var e error
+ if signed {
+ raw, e = release.SignRootEnvelope(sg, v)
+ } else {
+ raw, e = json.Marshal(v)
+ }
+ if e != nil {
+ t.Fatal(e)
+ }
+ tag, e := f.store.Put(ctx, k, raw, "")
+ if e != nil {
+ t.Fatal(e)
+ }
+ return tag
+ }
+ put(hostactors.Key, roster, true)
+ d := hostactors.Directory{Store: f.store, PublicKey: pub, Bucket: "bucket"}
+ c := NewBackfillController(f.store, d, func() time.Time { return f.now }, f.c.NewID)
+ r := f.record(t, "legacy.example.com")
+ var body map[string]any
+ if err = json.Unmarshal(r.Body, &body); err != nil {
+ t.Fatal(err)
+ }
+ body["acq"] = "ownership-acquisition"
+ put(r.Holder.Key, body, false)
+ r, err = ReadRecord(ctx, f.store, r.Holder.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ inc, err := r.Incarnation()
+ if err != nil {
+ t.Fatal(err)
+ }
+ put(domains.ClaimKey(r.Holder.Host), domains.Claim{Schema: 1, Host: r.Holder.Host, Account: "bab", App: "jam", Acquisition: "ownership-acquisition", RecordIncarnation: inc, OwnershipState: "committed"}, false)
+ if err = c.ReconcileClaims(ctx); err != nil {
+ t.Fatalf("unfrozen adoption: %v", err)
+ }
+ obs, err := hostclaim.Read(ctx, f.store, r.Holder.Host)
+ if err != nil || !obs.Absent {
+ t.Fatal(obs, err)
+ }
+ var prior hostclaim.Observed
+ if legacy {
+ nh := r.Holder
+ nh.Acquisition = ""
+ nh.Region = "ie"
+ prior, err = hostclaim.Update(ctx, f.store, obs, func(claim *hostclaim.Claim) error {
+ if err := hostclaim.Adopt(nh, "box-a", []string{"us"}, []string{"box-b"}, "legacy-discovery", f.c.NewID)(claim); err != nil {
+ return err
+ }
+ claim.Holder.Stamp = 17
+ claim.Holder.Generation = "legacy-generation"
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ p := BackfillPlan{V: 1, ID: "backfill", Phase: "adopting", DirectoryRevision: 1, CreatedAt: f.now, Reports: map[string]string{}, Hosts: map[string]BackfillHost{r.Holder.Host: {Key: r.Holder.Key, ETag: r.ETag, Acquisition: "ownership-acquisition", Region: "ie", Box: "box-a"}}}
+ for ref, b := range roster.Boxes {
+ report := CensusReport{Box: ref, Enrollment: b.Enrollment, BootID: "boot", Reservation: p.ID, DirectoryRevision: 1, CapturedAt: f.now, Complete: true, Installations: []CensusInstallation{}}
+ if ref == "box-a" || scenario == "two-owners" {
+ report.Installations = append(report.Installations, CensusInstallation{Host: r.Holder.Host, Account: "bab", App: "jam", Family: "instance", Key: r.Holder.Key, Incarnation: inc, Local: true, Route: true})
+ }
+ key, _ := reportKey(p.ID, ref)
+ body, e := json.Marshal(report)
+ if e != nil {
+ t.Fatal(e)
+ }
+ put(key, CensusEnvelope{Report: report, Signature: ed25519.Sign(boxKeys[ref], body)}, false)
+ raw, _, err := f.store.Get(ctx, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(raw)
+ p.Reports[ref] = hex.EncodeToString(sum[:])
+ }
+ put(BackfillKey, p, true)
+ qtag := put(HostsQuarantineKey, HostsQuarantine{V: 1, Hosts: map[string]string{r.Holder.Host: "conflict"}}, false)
+ if err = c.ReconcileClaims(ctx); !errors.Is(err, ErrPending) {
+ t.Fatalf("quarantine: %v", err)
+ }
+ raw, _ := json.Marshal(HostsQuarantine{V: 1, Hosts: map[string]string{}})
+ if _, err = f.store.Put(ctx, HostsQuarantineKey, raw, qtag); err != nil {
+ t.Fatal(err)
+ }
+
+ // A digest listed by a root manifest cannot turn a forged box report into
+ // an authenticated census. Test the verifier before its cache is populated.
+ reportPath, _ := reportKey(p.ID, "box-a")
+ authentic, reportTag, e := f.store.Get(ctx, reportPath)
+ if e != nil {
+ t.Fatal(e)
+ }
+ var forged CensusEnvelope
+ if e = json.Unmarshal(authentic, &forged); e != nil {
+ t.Fatal(e)
+ }
+ forged.Signature[0] ^= 1
+ forgedRaw, _ := json.Marshal(forged)
+ forgedTag, e := f.store.Put(ctx, reportPath, forgedRaw, reportTag)
+ if e != nil {
+ t.Fatal(e)
+ }
+ sum := sha256.Sum256(forgedRaw)
+ p.Reports["box-a"] = hex.EncodeToString(sum[:])
+ _, manifestTag, e := f.store.Get(ctx, BackfillKey)
+ if e != nil {
+ t.Fatal(e)
+ }
+ signed, _ := release.SignRootEnvelope(sg, p)
+ manifestTag, e = f.store.Put(ctx, BackfillKey, signed, manifestTag)
+ if e != nil {
+ t.Fatal(e)
+ }
+ if e = c.ReconcileClaims(ctx); e == nil {
+ t.Fatal("accepted forged census")
+ }
+ if _, e = f.store.Put(ctx, reportPath, authentic, forgedTag); e != nil {
+ t.Fatal(e)
+ }
+ sum = sha256.Sum256(authentic)
+ p.Reports["box-a"] = hex.EncodeToString(sum[:])
+ signed, _ = release.SignRootEnvelope(sg, p)
+ if _, e = f.store.Put(ctx, BackfillKey, signed, manifestTag); e != nil {
+ t.Fatal(e)
+ }
+ if scenario == "two-owners" || scenario == "inactive-owner" || scenario == "wrong-region" {
+ if err = c.ReconcileClaims(ctx); !errors.Is(err, ErrIdentity) {
+ t.Fatalf("ineligible census owner: %v", err)
+ }
+ obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host)
+ if err != nil || !obs.Absent {
+ t.Fatalf("ineligible owner adopted: %+v %v", obs, err)
+ }
+ return
+ }
+ if err = c.ReconcileClaims(ctx); err != nil {
+ t.Fatal(err)
+ }
+ obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if obs.Absent || obs.Claim.Holder.Acquisition != "ownership-acquisition" || obs.Claim.Op.Exec != "box-a" || obs.Claim.Touched.Complete || len(obs.Claim.Obligations) != 2 {
+ t.Fatalf("bad backfill: %+v", obs.Claim)
+ }
+ wantDestinations := map[string]hostclaim.Obligation{
+ "park-replica": {Region: "us", Owner: "region:us"},
+ "park-local": {Box: "box-b", Owner: "box:box-b"},
+ }
+ for _, obligation := range obs.Claim.Obligations {
+ want, ok := wantDestinations[obligation.Kind]
+ if !ok || obligation.Region != want.Region || obligation.Box != want.Box || obligation.Owner != want.Owner {
+ t.Fatalf("wrong discovery destination: %+v; want %+v", obligation, want)
+ }
+ delete(wantDestinations, obligation.Kind)
+ }
+ if len(wantDestinations) != 0 {
+ t.Fatalf("missing discovery destinations: %+v", wantDestinations)
+ }
+ if legacy {
+ want := *prior.Claim
+ holder := *want.Holder
+ holder.Acquisition = "ownership-acquisition"
+ want.Holder = &holder
+ want.Gen++
+ if obs.ETag == prior.ETag || !reflect.DeepEqual(obs.Claim, &want) {
+ t.Fatalf("legacy repair changed more than acquisition and claim generation: got %+v (holder %+v), want %+v (holder %+v)", obs.Claim, obs.Claim.Holder, &want, want.Holder)
+ }
+ }
+ before := obs.ETag
+ if err = c.ReconcileClaims(ctx); err != nil {
+ t.Fatal(err)
+ }
+ obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host)
+ if err != nil || obs.ETag != before {
+ t.Fatal("non-idempotent", err)
+ }
+}
+func TestReconcileDiscardPartialCensus(t *testing.T) {
+ f := newFixture(t)
+ r := f.record(t, "legacy.example.com")
+ if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
+ t.Fatal(err)
+ }
+ fail := errors.New("second listing failed")
+ f.store.afterList = func(prefix string, keys []string) error {
+ if strings.Contains(prefix, "sidecars/") {
+ return fail
+ }
+ return nil
+ }
+ calls := 0
+ err := ReconcileClaims(f.ctx, f.c, func(context.Context, string) (hostclaim.Observed, error) { calls++; return hostclaim.Observed{}, nil })
+ if !errors.Is(err, fail) || calls != 0 {
+ t.Fatal(err, calls)
+ }
+}
+
+func TestClaimDeleteDoorNeverAdopts(t *testing.T) {
+ f := newFixture(t)
+ r := f.record(t, "legacy.example.com")
+ if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
+ t.Fatal(err)
+ }
+ inc, err := r.Incarnation()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = f.c.BeginRetire(f.ctx, r.Holder.Key, "test", inc); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+ obs, err := hostclaim.Read(f.ctx, f.store, r.Holder.Host)
+ if err != nil || !obs.Absent {
+ t.Fatalf("door adopted: %+v %v", obs, err)
+ }
+}
+
+func TestBackfillAuthorityCannotGainMutationMethods(t *testing.T) {
+ c := NewBackfillController(nil, hostactors.Directory{}, time.Now, func() string { return "id" })
+ if _, ok := c.Authority.(interface {
+ StartCreate(context.Context, hostclaim.Observed, Record, string, string) (hostclaim.Observed, error)
+ }); ok {
+ t.Fatal("backfill gained create authority")
+ }
+ if _, ok := c.Authority.(interface {
+ StartDeleteClaim(context.Context, hostclaim.Observed, Record, string) (hostclaim.Observed, error)
+ }); ok {
+ t.Fatal("backfill gained delete authority")
+ }
+ if _, ok := c.Authority.(interface {
+ RecordEffects(context.Context, hostclaim.Observed, Record) error
+ }); ok {
+ t.Fatal("backfill gained effect authority")
+ }
+}
+
+func TestReconcileBackfillCensusOwnerEligibility(t *testing.T) {
+ for _, scenario := range []string{"two-box-region", "two-owners", "inactive-owner", "wrong-region"} {
+ t.Run(scenario, func(t *testing.T) {
+ testReconcileFrozenBackfill(t, false, scenario)
+ })
+ }
+}
+
+func TestReconcileBackfillPlanPhases(t *testing.T) {
+ for _, phase := range []string{"absent", "collecting", "done", "future", "adopting", "bad-signature", "read-error"} {
+ t.Run(phase, func(t *testing.T) {
+ f := newFixture(t)
+ pub, key, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if phase != "absent" && phase != "read-error" {
+ p := BackfillPlan{V: 1, ID: "backfill", Phase: phase, CreatedAt: f.now, Hosts: map[string]BackfillHost{}}
+ raw, err := release.SignRootEnvelope(release.RootSigner(key), p)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = f.store.Put(f.ctx, BackfillKey, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if phase == "bad-signature" {
+ pub = make(ed25519.PublicKey, ed25519.PublicKeySize)
+ }
+ if phase == "read-error" {
+ f.store.failGet = BackfillKey
+ }
+ c := NewBackfillController(f.store, hostactors.Directory{Store: f.store, PublicKey: pub, Bucket: "bucket"}, f.c.Now, f.c.NewID)
+ lists, adopts := 0, 0
+ f.store.afterList = func(string, []string) error { lists++; return nil }
+ err = ReconcileClaims(f.ctx, c, func(context.Context, string) (hostclaim.Observed, error) { adopts++; return hostclaim.Observed{}, nil })
+ wantError := phase == "adopting" || phase == "bad-signature" || phase == "read-error"
+ if (err != nil) != wantError || lists != 0 || adopts != 0 {
+ t.Fatalf("phase %s: err=%v lists=%d adopts=%d", phase, err, lists, adopts)
+ }
+ if !wantError && phase != "absent" {
+ if _, _, _, _, err := c.Authority.Adoption(f.ctx, Record{}); err == nil {
+ t.Fatal("inactive plan granted direct adoption")
+ }
+ }
+ })
+ }
+}
diff --git a/internal/hostops/operations.go b/internal/hostops/operations.go
new file mode 100644
index 000000000..8313b80e6
--- /dev/null
+++ b/internal/hostops/operations.go
@@ -0,0 +1,304 @@
+package hostops
+
+import (
+ "context"
+ "errors"
+ "os"
+ "slices"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// These callbacks are authority providers, not optional best-effort hints.
+// Production construction waits for the decisions named in this plan.
+type Authority interface {
+ Create(context.Context, hostclaim.Observed, Record, string) error
+ Executor(context.Context, hostclaim.Observed, Record) (string, error)
+ StartDelete(context.Context, hostclaim.Observed, Record) error
+ FirstPhase(context.Context, hostclaim.Observed) error
+ Inventory(context.Context, hostclaim.Observed) ([]string, []string, error)
+ Adoption(context.Context, Record) (region, box string, otherRegions, otherBoxes []string, err error)
+ PurgeRetirement(context.Context, hostclaim.Observed) error
+}
+
+type Controller struct {
+ Store release.Store
+ Authority Authority
+ Now func() time.Time
+ NewID func() string
+}
+
+func (c Controller) ready() error {
+ if c.Store == nil || c.Authority == nil || c.Now == nil || c.NewID == nil {
+ return ErrContract
+ }
+ return nil
+}
+
+// Attempt consumes the claim observation taken BEFORE app registration.
+// Create's caller owns registration renewal/removal. A failed CAS is never retried here.
+func (c Controller) Attempt(ctx context.Context, before hostclaim.Observed, r Record, id, by string) (hostclaim.Observed, error) {
+ if err := c.ready(); err != nil {
+ return hostclaim.Observed{}, err
+ }
+ if before.Host != r.Holder.Host || r.Holder.CreatedAt.IsZero() || id == "" {
+ return hostclaim.Observed{}, ErrIdentity
+ }
+ // Last pre-CAS read of registration, open admission and purge reservation.
+ if err := c.Authority.Create(ctx, before, r, id); err != nil {
+ return hostclaim.Observed{}, err
+ }
+ obs, err := hostclaim.Update(ctx, c.Store, before, hostclaim.BeginCreate(r.Holder, r.Body, id, by, c.Now()))
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ // No raw record effect precedes the durable claim intent.
+ if _, err = c.Store.Put(ctx, r.Holder.Key, r.Body, ""); err != nil {
+ if !errors.Is(err, release.ErrCASConflict) {
+ return obs, err
+ }
+ existing, rerr := ReadRecord(ctx, c.Store, r.Holder.Key)
+ if rerr != nil {
+ return obs, rerr
+ }
+ if !matches(existing, obs.Claim.Holder) {
+ removed, rerr := RemoveOrphan(ctx, c.Store, existing)
+ if rerr != nil {
+ return obs, rerr
+ }
+ if !removed {
+ return obs, ErrPending
+ }
+ if _, err = c.Store.Put(ctx, r.Holder.Key, r.Body, ""); err != nil {
+ return obs, err
+ }
+ }
+ }
+ return c.Complete(ctx, obs.Host, id)
+}
+
+func (c Controller) Complete(ctx context.Context, host, id string) (hostclaim.Observed, error) {
+ if err := c.ready(); err != nil {
+ return hostclaim.Observed{}, err
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, host)
+ if err != nil {
+ return obs, err
+ }
+ if obs.Absent || obs.Claim.Op.Kind != hostclaim.KindCreating || obs.Claim.Op.ID != id {
+ return obs, ErrPending
+ }
+ r, err := ReadRecord(ctx, c.Store, obs.Claim.Holder.Key)
+ if err != nil {
+ return obs, err
+ }
+ if !matches(r, obs.Claim.Holder) {
+ return obs, ErrIdentity
+ }
+ box, err := c.Authority.Executor(ctx, obs, r)
+ if err != nil {
+ return obs, err
+ }
+ return hostclaim.Update(ctx, c.Store, obs, hostclaim.CompleteCreate(id, box))
+}
+
+func (c Controller) RecoverCreate(ctx context.Context, host string) error {
+ if err := c.ready(); err != nil {
+ return err
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, host)
+ if err != nil {
+ return err
+ }
+ if obs.Absent || obs.Claim.Op.Kind != hostclaim.KindCreating {
+ return nil
+ }
+ if obs.Claim.Op.StartedAt.IsZero() || !c.Now().After(obs.Claim.Op.StartedAt.Add(5*time.Minute)) {
+ return ErrPending
+ }
+ r, err := ReadRecord(ctx, c.Store, obs.Claim.Holder.Key)
+ if errors.Is(err, os.ErrNotExist) {
+ _, err = hostclaim.Update(ctx, c.Store, obs, hostclaim.CancelCreate(obs.Claim.Op.ID, c.Now()))
+ return err
+ }
+ if err != nil {
+ return err
+ }
+ if !matches(r, obs.Claim.Holder) {
+ return ErrPending
+ }
+ _, err = c.Complete(ctx, host, obs.Claim.Op.ID)
+ return err
+}
+
+// Adopt is the shared IO wrapper around the one hostclaim.Adopt transition.
+func (c Controller) Adopt(ctx context.Context, r Record) (hostclaim.Observed, error) {
+ if err := c.ready(); err != nil {
+ return hostclaim.Observed{}, err
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, r.Holder.Host)
+ if err != nil {
+ return obs, err
+ }
+ if !obs.Absent {
+ if matches(r, obs.Claim.Holder) {
+ return obs, nil
+ }
+ if obs.Claim.Holder == nil || obs.Claim.Holder.Acquisition != "" || r.Holder.Acquisition == "" {
+ return obs, ErrIdentity
+ }
+ // #232 also repairs already-claimed legacy holders while globally
+ // frozen. Adoption authority verifies the exact manifest/ownership.
+ prior := *obs.Claim.Holder
+ prior.Acquisition = r.Holder.Acquisition
+ if !matches(r, &prior) {
+ return obs, ErrIdentity
+ }
+ _, box, _, _, err := c.Authority.Adoption(ctx, r)
+ if err != nil {
+ return obs, err
+ }
+ if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Exec != box || !slices.Contains(obs.Claim.Touched.Boxes, box) {
+ return obs, ErrIdentity
+ }
+ fresh, err := ReadRecord(ctx, c.Store, r.Holder.Key)
+ if err != nil {
+ return obs, err
+ }
+ if fresh.ETag != r.ETag {
+ return obs, ErrPending
+ }
+ return hostclaim.Update(ctx, c.Store, obs, func(next *hostclaim.Claim) error {
+ if next.Holder == nil || next.Holder.Acquisition != "" {
+ return ErrPending
+ }
+ next.Holder.Acquisition = r.Holder.Acquisition
+ return nil
+ })
+ }
+ region, box, regions, boxes, err := c.Authority.Adoption(ctx, r)
+ if err != nil {
+ return obs, err
+ }
+ // Re-read the exact record after discovery; never adopt a stale listing body.
+ fresh, err := ReadRecord(ctx, c.Store, r.Holder.Key)
+ if err != nil {
+ return obs, err
+ }
+ if fresh.ETag != r.ETag {
+ return obs, ErrPending
+ }
+ n := fresh.Holder
+ n.Region = region
+ return hostclaim.Update(ctx, c.Store, obs, hostclaim.Adopt(n, box, regions, boxes, c.NewID(), c.NewID))
+}
+
+// BeginRetire only starts the claim operation. Its successful return does not
+// mean the record is gone or the host is free; the caller must run/retry Retire.
+func (c Controller) BeginRetire(ctx context.Context, key, by, expectedInc string) (hostclaim.Observed, error) {
+ if err := c.ready(); err != nil {
+ return hostclaim.Observed{}, err
+ }
+ r, err := ReadRecord(ctx, c.Store, key)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ inc, ierr := r.Incarnation()
+ if ierr != nil {
+ return hostclaim.Observed{}, ierr
+ }
+ if expectedInc == "" || inc != expectedInc {
+ return hostclaim.Observed{}, ErrIdentity
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, r.Holder.Host)
+ if err != nil {
+ return obs, err
+ }
+ if obs.Absent {
+ return obs, ErrPending
+ }
+ if !matches(r, obs.Claim.Holder) {
+ return obs, ErrIdentity
+ }
+ if err := c.Authority.StartDelete(ctx, obs, r); err != nil {
+ return obs, err
+ }
+ regions, boxes, err := c.Authority.Inventory(ctx, obs)
+ if err != nil {
+ return obs, err
+ }
+ return hostclaim.Update(ctx, c.Store, obs, hostclaim.BeginDelete(c.NewID(), by, r.ETag, c.Now(), regions, boxes))
+}
+
+func (c Controller) Retire(ctx context.Context, host, id string) error {
+ if err := c.ready(); err != nil {
+ return err
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, host)
+ if err != nil {
+ return err
+ }
+ if obs.Absent || obs.Claim.Op.ID != id {
+ return ErrPending
+ }
+ op := obs.Claim.Op
+ if op.Kind != hostclaim.KindDeleting && op.Kind != hostclaim.KindPurging {
+ return ErrPending
+ }
+ if obs.Claim.Holder == nil || op.Intent["key"] != obs.Claim.Holder.Key || op.Intent["etag"] == "" {
+ return ErrIdentity
+ }
+ if err := c.Authority.FirstPhase(ctx, obs); err != nil {
+ return err
+ }
+ if op.Kind == hostclaim.KindPurging {
+ if err := c.Authority.PurgeRetirement(ctx, obs); err != nil {
+ return err
+ }
+ }
+ regions, boxes, err := c.Authority.Inventory(ctx, obs)
+ if err != nil {
+ return err
+ }
+ // Validate record identity before deleting even the stored version.
+ r, rerr := ReadRecord(ctx, c.Store, op.Intent["key"])
+ if rerr != nil && !errors.Is(rerr, os.ErrNotExist) {
+ return rerr
+ }
+ if rerr == nil {
+ if r.ETag != op.Intent["etag"] || !matches(r, obs.Claim.Holder) {
+ return ErrPending
+ }
+ if err := release.DeleteIfMatch(ctx, c.Store, r.Holder.Key, r.ETag); err != nil {
+ return err
+ }
+ }
+ // A success/412 alone never certifies absence. Re-read before parking.
+ _, _, err = c.Store.Get(ctx, op.Intent["key"])
+ if !errors.Is(err, os.ErrNotExist) {
+ if err != nil {
+ return err
+ }
+ return ErrPending
+ }
+ // The old observation binds the entire sequence to this operation.
+ _, err = hostclaim.Update(ctx, c.Store, obs, hostclaim.ToParking(id, regions, boxes, c.NewID))
+ return err
+}
+
+func (c Controller) Free(ctx context.Context, host, id string) error {
+ if err := c.ready(); err != nil {
+ return err
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, host)
+ if err != nil {
+ return err
+ }
+ if obs.Absent {
+ return ErrPending
+ }
+ _, err = hostclaim.Update(ctx, c.Store, obs, hostclaim.FreeAfterPark(id))
+ return err
+}
diff --git a/internal/hostops/operations_test.go b/internal/hostops/operations_test.go
new file mode 100644
index 000000000..42ebf9d11
--- /dev/null
+++ b/internal/hostops/operations_test.go
@@ -0,0 +1,629 @@
+package hostops
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "reflect"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type authority struct{ err error }
+
+func (a *authority) Create(context.Context, hostclaim.Observed, Record, string) error { return a.err }
+func (a *authority) Executor(context.Context, hostclaim.Observed, Record) (string, error) {
+ return "box-a", a.err
+}
+func (a *authority) StartDelete(context.Context, hostclaim.Observed, Record) error { return a.err }
+func (a *authority) FirstPhase(context.Context, hostclaim.Observed) error { return a.err }
+func (a *authority) Inventory(context.Context, hostclaim.Observed) ([]string, []string, error) {
+ return []string{"ie", "us"}, []string{"box-a", "box-b"}, a.err
+}
+func (a *authority) Adoption(context.Context, Record) (string, string, []string, []string, error) {
+ return "ie", "box-a", []string{"us"}, []string{"box-b"}, a.err
+}
+func (a *authority) PurgeRetirement(context.Context, hostclaim.Observed) error { return a.err }
+
+type faultStore struct {
+ release.Store
+ failGet string
+ afterList func(string, []string) error
+ beforePut func(string) error
+ beforeDelete func(string, string) error
+ bareDeletes int
+}
+
+func (s *faultStore) List(ctx context.Context, prefix string) ([]string, error) {
+ keys, err := s.Store.List(ctx, prefix)
+ if err == nil && s.afterList != nil {
+ err = s.afterList(prefix, keys)
+ }
+ return keys, err
+}
+
+func (s *faultStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ if key == s.failGet {
+ return nil, "", errors.New("read failed")
+ }
+ return s.Store.Get(ctx, key)
+}
+func (s *faultStore) Put(ctx context.Context, key string, b []byte, e string) (string, error) {
+ if s.beforePut != nil {
+ if err := s.beforePut(key); err != nil {
+ return "", err
+ }
+ }
+ return s.Store.Put(ctx, key, b, e)
+}
+func (s *faultStore) Delete(context.Context, string) error {
+ s.bareDeletes++
+ return errors.New("bare delete forbidden")
+}
+func (s *faultStore) DeleteIfMatch(ctx context.Context, key, e string) error {
+ if s.beforeDelete != nil {
+ if err := s.beforeDelete(key, e); err != nil {
+ return err
+ }
+ }
+ return release.DeleteIfMatch(ctx, s.Store, key, e)
+}
+
+type fixture struct {
+ ctx context.Context
+ store *faultStore
+ c Controller
+ auth *authority
+ now time.Time
+ serial int
+}
+
+func newFixture(t *testing.T) *fixture {
+ t.Helper()
+ f := &fixture{ctx: context.Background(), now: time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC), auth: &authority{}}
+ f.store = &faultStore{Store: release.NewDirStore(t.TempDir())}
+ f.c = Controller{Store: f.store, Authority: f.auth, Now: func() time.Time { return f.now }, NewID: func() string { f.serial++; return fmt.Sprintf("id-%d", f.serial) }}
+ return f
+}
+func (f *fixture) record(t *testing.T, host string) Record {
+ t.Helper()
+ r, err := PrepareInstance(provision.Record{Schema: 1, Account: "bab", App: "jam", Host: host, Slug: "shop", Channel: "edge", Region: "ie"}, f.now, "ie")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return r
+}
+func (f *fixture) serving(t *testing.T, host string) hostclaim.Observed {
+ t.Helper()
+ r := f.record(t, host)
+ before, err := hostclaim.Read(f.ctx, f.store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ obs, err := f.c.Attempt(f.ctx, before, r, f.c.NewID(), "test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return obs
+}
+func TestAttemptUsesPreRegistrationObservation(t *testing.T) {
+ f := newFixture(t)
+ for _, host := range []string{"one.bab.example.com", "two.bab.example.com"} {
+ before, err := hostclaim.Read(f.ctx, f.store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fenced, err := hostclaim.Update(f.ctx, f.store, before, FencePurge("purge", f.now))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = hostclaim.Update(f.ctx, f.store, fenced, ReleasePurgeFence("purge")); err != nil {
+ t.Fatal(err)
+ }
+ r := f.record(t, host)
+ if _, err = f.c.Attempt(f.ctx, before, r, "paused", "test"); !errors.Is(err, hostclaim.ErrConflict) {
+ t.Fatalf("late claim: %v", err)
+ }
+ if _, _, err = f.store.Get(f.ctx, r.Holder.Key); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("record landed: %v", err)
+ }
+ }
+ if f.store.bareDeletes != 0 {
+ t.Fatal("deleted claim")
+ }
+}
+func TestRecoverCreateAbsentAndLateOrphan(t *testing.T) {
+ f := newFixture(t)
+ host := "shop.bab.example.com"
+ r := f.record(t, host)
+ before, _ := hostclaim.Read(f.ctx, f.store, host)
+ if _, err := hostclaim.Update(f.ctx, f.store, before, hostclaim.BeginCreate(r.Holder, r.Body, "create", "test", f.now)); err != nil {
+ t.Fatal(err)
+ }
+ f.now = f.now.Add(5 * time.Minute)
+ if err := f.c.RecoverCreate(f.ctx, host); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+ f.now = f.now.Add(time.Nanosecond)
+ if err := f.c.RecoverCreate(f.ctx, host); err != nil {
+ t.Fatal(err)
+ }
+ obs, _ := hostclaim.Read(f.ctx, f.store, host)
+ if !obs.Claim.Free() || len(obs.Claim.CancelledCreates) != 1 || len(obs.Claim.RetiredIncarnations) != 1 {
+ t.Fatalf("cancel: %+v", obs.Claim)
+ }
+ f.now = f.now.Add(2 * time.Hour)
+ if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
+ t.Fatal(err)
+ }
+ late, err := ReadRecord(f.ctx, f.store, r.Holder.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ removed, err := RemoveOrphan(f.ctx, f.store, late)
+ if err != nil || !removed {
+ t.Fatalf("late orphan: %t %v", removed, err)
+ }
+}
+func TestCreateRecoveryReadErrorsNeverCancel(t *testing.T) {
+ f := newFixture(t)
+ r := f.record(t, "shop.bab.example.com")
+ before, _ := hostclaim.Read(f.ctx, f.store, r.Holder.Host)
+ obs, err := hostclaim.Update(f.ctx, f.store, before, hostclaim.BeginCreate(r.Holder, r.Body, "create", "test", f.now))
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.now = f.now.Add(6 * time.Minute)
+ f.store.failGet = r.Holder.Key
+ if err := f.c.RecoverCreate(f.ctx, r.Holder.Host); err == nil {
+ t.Fatal("read error accepted")
+ }
+ got, _ := hostclaim.Read(f.ctx, f.store, r.Holder.Host)
+ if got.ETag != obs.ETag {
+ t.Fatal("acted after read error")
+ }
+ f.store.failGet = ""
+ if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.c.RecoverCreate(f.ctx, r.Holder.Host); err != nil {
+ t.Fatal(err)
+ }
+ got, _ = hostclaim.Read(f.ctx, f.store, r.Holder.Host)
+ if got.Claim.Op.Kind != hostclaim.KindServing || got.Claim.Holder.Stamp != got.Claim.Gen {
+ t.Fatal("not completed")
+ }
+}
+func TestRetireWaitsForExactRecordAndAuthority(t *testing.T) {
+ f := newFixture(t)
+ obs := f.serving(t, "shop.bab.example.com")
+ key := obs.Claim.Holder.Key
+ deleting, err := f.c.BeginRetire(f.ctx, key, "test", obs.Claim.Holder.Incarnation)
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.auth.err = ErrPending
+ if err := f.c.Retire(f.ctx, obs.Host, deleting.Claim.Op.ID); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+ if _, _, err := f.store.Get(f.ctx, key); err != nil {
+ t.Fatal("record deleted before barrier")
+ }
+ f.auth.err = nil
+ f.store.beforeDelete = func(k, e string) error { return release.ErrCASConflict }
+ if err := f.c.Retire(f.ctx, obs.Host, deleting.Claim.Op.ID); !errors.Is(err, release.ErrCASConflict) {
+ t.Fatal(err)
+ }
+ current, _ := hostclaim.Read(f.ctx, f.store, obs.Host)
+ if current.Claim.Op.Kind != hostclaim.KindDeleting {
+ t.Fatal("parked after 412")
+ }
+ f.store.beforeDelete = nil
+ if err := f.c.Retire(f.ctx, obs.Host, deleting.Claim.Op.ID); err != nil {
+ t.Fatal(err)
+ }
+ current, _ = hostclaim.Read(f.ctx, f.store, obs.Host)
+ if current.Claim.Op.Kind != hostclaim.KindParking || len(current.Claim.Obligations) != 2 {
+ t.Fatalf("parking: %+v", current.Claim)
+ }
+ if err := f.c.Free(f.ctx, obs.Host, deleting.Claim.Op.ID); err == nil {
+ t.Fatal("freed with chores")
+ }
+ for _, ob := range current.Claim.Obligations {
+ var err error
+ current, err = hostclaim.Update(f.ctx, f.store, current, hostclaim.CompleteObligation(ob.Op, ob.ID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := f.c.Free(f.ctx, obs.Host, deleting.Claim.Op.ID); err != nil {
+ t.Fatal(err)
+ }
+ if f.store.bareDeletes != 0 {
+ t.Fatal("bare delete")
+ }
+}
+func TestAdoptionIsNotOrphanAndHasDiscovery(t *testing.T) {
+ f := newFixture(t)
+ r := f.record(t, "shop.bab.example.com")
+ if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
+ t.Fatal(err)
+ }
+ read, err := ReadRecord(f.ctx, f.store, r.Holder.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if removed, err := RemoveOrphan(f.ctx, f.store, read); err != nil || removed {
+ t.Fatal("legacy treated as orphan")
+ }
+ obs, err := f.c.Adopt(f.ctx, read)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if obs.Claim.Touched.Complete || len(obs.Claim.Obligations) != 2 || obs.Claim.Holder.Location != "instances/"+r.Holder.Host || obs.Claim.Holder.WriterBox != "box-a" {
+ t.Fatalf("adopt: %+v", obs.Claim)
+ }
+ if _, err := f.c.BeginRetire(f.ctx, r.Holder.Key, "test", obs.Claim.Holder.Incarnation); err == nil {
+ t.Fatal("delete overlapped discovery")
+ }
+}
+func TestPurgeJournalPhasesAndFenceRetention(t *testing.T) {
+ f := newFixture(t)
+ p, _, err := ReservePurge(f.ctx, f.store, "bab", "jam", "purge")
+ if err != nil || p.Phase != "reserved" {
+ t.Fatal(err)
+ }
+ if _, _, err := ReservePurge(f.ctx, f.store, "bab", "jam", "other"); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+ if err := RemovePurge(f.ctx, f.store, "bab", "jam", "purge"); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+ if err := AdvancePurge(f.ctx, f.store, "bab", "jam", "purge", "reserved", "done"); err == nil {
+ t.Fatal("skipped phases")
+ }
+ for i := 0; i < len(phases)-1; i++ {
+ if err := AdvancePurge(f.ctx, f.store, "bab", "jam", "purge", phases[i], phases[i+1]); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := RemovePurge(f.ctx, f.store, "bab", "jam", "purge"); err != nil {
+ t.Fatal(err)
+ }
+}
+func TestPurgeRetiresUnderOwnOperation(t *testing.T) {
+ f := newFixture(t)
+ obs := f.serving(t, "shop.bab.example.com")
+ rec, err := ReadRecord(f.ctx, f.store, obs.Claim.Holder.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ obs, err = hostclaim.Update(f.ctx, f.store, obs, BeginPurge("purge", "test", f.now, []string{"ie"}, []string{"box-a"}))
+ if err != nil {
+ t.Fatal(err)
+ }
+ obs, err = hostclaim.Update(f.ctx, f.store, obs, PurgeRecordIntent("purge", rec.Holder.Key, rec.ETag))
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.auth.err = ErrPending
+ if err := f.c.Retire(f.ctx, obs.Host, "purge"); !errors.Is(err, ErrPending) {
+ t.Fatal(err)
+ }
+ f.auth.err = nil
+ if err := f.c.Retire(f.ctx, obs.Host, "purge"); err != nil {
+ t.Fatal(err)
+ }
+ got, _ := hostclaim.Read(f.ctx, f.store, obs.Host)
+ if got.Claim.Op.Kind != hostclaim.KindParking || got.Claim.Op.ID != "purge" {
+ t.Fatal("purge identity lost")
+ }
+}
+func TestPrepareRejectsCallerTimeAndSidecarIdentity(t *testing.T) {
+ f := newFixture(t)
+ if _, err := PrepareInstance(provision.Record{CreatedAt: f.now}, f.now, "ie"); err == nil {
+ t.Fatal("accepted caller time")
+ }
+ r, err := PrepareSidecar(provision.SidecarRecord{Account: "bab", App: "jam", Name: "job", Host: provision.SidecarHost("job", "jam", "bab"), Region: "ie"}, f.now, "ie")
+ if err != nil {
+ t.Fatal(err)
+ }
+ before, _ := hostclaim.Read(f.ctx, f.store, r.Holder.Host)
+ obs, err := f.c.Attempt(f.ctx, before, r, "sidecar", "test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if obs.Claim.Holder.Family != "sidecar" {
+ t.Fatal("sidecar became instance")
+ }
+}
+func TestRecoveryFreshReadsAndRequiredHooks(t *testing.T) {
+ f := newFixture(t)
+ if err := (Recovery{Controller: f.c}).Pass(f.ctx); !errors.Is(err, ErrContract) {
+ t.Fatal(err)
+ }
+ for _, tc := range []struct {
+ name string
+ replace bool
+ wantCalls int
+ }{
+ {name: "advanced purge", replace: true, wantCalls: 1},
+ {name: "released purge", wantCalls: 0},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newFixture(t)
+ host := "shop.bab.example.com"
+ before, err := hostclaim.Read(f.ctx, f.store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ before, err = hostclaim.Update(f.ctx, f.store, before, FencePurge("purge-old", f.now))
+ if err != nil {
+ t.Fatal(err)
+ }
+ startedAt := f.now
+ f.now = f.now.Add(6 * time.Minute)
+ key, err := hostclaim.Key(host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var fresh hostclaim.Observed
+ listCalls := 0
+ f.store.afterList = func(prefix string, keys []string) error {
+ listCalls++
+ if prefix != "control/hosts/" || len(keys) != 1 || keys[0] != key {
+ t.Fatalf("discovery = %q %v, want the populated claim", prefix, keys)
+ }
+ var err error
+ fresh, err = hostclaim.Update(f.ctx, f.store.Store, before, ReleasePurgeFence("purge-old"))
+ if err == nil && tc.replace {
+ fresh, err = hostclaim.Update(f.ctx, f.store.Store, fresh, FencePurge("purge-new", startedAt))
+ }
+ return err
+ }
+ purgeCalls, recordSweeps, purgeSweeps, admissionSweeps := 0, 0, 0, 0
+ r := Recovery{
+ Controller: f.c,
+ Purge: func(_ context.Context, got hostclaim.Observed) error {
+ purgeCalls++
+ if !tc.replace {
+ t.Fatal("dispatched a purge released after discovery")
+ }
+ if !reflect.DeepEqual(got, fresh) {
+ t.Fatalf("dispatched stale claim: gen=%d op=%q etag=%q; want gen=%d op=%q etag=%q", got.Claim.Gen, got.Claim.Op.ID, got.ETag, fresh.Claim.Gen, fresh.Claim.Op.ID, fresh.ETag)
+ }
+ return nil
+ },
+ Placement: func(context.Context, hostclaim.Observed) error {
+ t.Fatal("dispatched placement for a purge claim")
+ return nil
+ },
+ SweepRecords: func(context.Context) error { recordSweeps++; return nil },
+ SweepPurges: func(context.Context) error { purgeSweeps++; return nil },
+ SweepAdmissions: func(context.Context) error { admissionSweeps++; return nil },
+ }
+ if err := r.Pass(f.ctx); err != nil {
+ t.Fatal(err)
+ }
+ if listCalls != 1 || fresh.Claim.Gen <= before.Claim.Gen || fresh.ETag == before.ETag {
+ t.Fatal("claim did not advance between discovery and recovery")
+ }
+ if purgeCalls != tc.wantCalls {
+ t.Fatalf("purge calls = %d, want %d", purgeCalls, tc.wantCalls)
+ }
+ if recordSweeps != 1 || purgeSweeps != 1 || admissionSweeps != 1 {
+ t.Fatalf("sweeps: records=%d purges=%d admissions=%d; want one each", recordSweeps, purgeSweeps, admissionSweeps)
+ }
+ })
+ }
+}
+
+func TestAppsPreserveLegacyPrefixesAndFindClaimlessPurge(t *testing.T) {
+ f := newFixture(t)
+ for key, body := range map[string][]byte{
+ release.LegacyChannelKey("my-app", "stable"): []byte("legacy channel"),
+ release.LegacyChannelKey("myapp", "stable"): []byte("legacy channel"),
+ release.LegacyManifestKey("myapp", "v1"): []byte("legacy manifest"),
+ release.LegacyArtifactKey("myapp", "v1", "app"): []byte("legacy artifact"),
+ } {
+ if _, err := f.store.Put(f.ctx, key, body, ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if _, _, err := ReservePurge(f.ctx, f.store, "bab", "doomed", "purge"); err != nil {
+ t.Fatal(err)
+ }
+ hosts, err := ListClaims(f.ctx, f.store)
+ if err != nil || len(hosts) != 0 {
+ t.Fatalf("claims = %v, %v; want none", hosts, err)
+ }
+ if err := f.c.SweepAllRecords(f.ctx); err != nil {
+ t.Fatalf("record sweep with legacy prefixes: %v", err)
+ }
+ var replayed []App
+ err = SweepAllPurges(f.ctx, f.store, func(_ context.Context, app App, p Purge) error {
+ if p.ID != "purge" {
+ t.Fatalf("purge ID = %q", p.ID)
+ }
+ replayed = append(replayed, app)
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(replayed) != 1 || replayed[0] != (App{Account: "bab", Name: "doomed"}) {
+ t.Fatalf("replayed = %+v", replayed)
+ }
+}
+
+func TestOrphanConditionalDeleteCannotEraseReplacement(t *testing.T) {
+ f := newFixture(t)
+ r := f.record(t, "shop.bab.example.com")
+ if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
+ t.Fatal(err)
+ }
+ old, err := ReadRecord(f.ctx, f.store, r.Holder.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ absent, _ := hostclaim.Read(f.ctx, f.store, r.Holder.Host)
+ if _, err := hostclaim.Update(f.ctx, f.store, absent, func(*hostclaim.Claim) error { return nil }); err != nil {
+ t.Fatal(err)
+ }
+ replacement := append([]byte(nil), r.Body...)
+ replacement = append(replacement, ' ')
+ f.store.beforeDelete = func(key, etag string) error {
+ _, err := f.store.Store.Put(f.ctx, key, replacement, etag)
+ return err
+ }
+ if removed, err := RemoveOrphan(f.ctx, f.store, old); removed || !errors.Is(err, release.ErrCASConflict) {
+ t.Fatalf("removed=%t err=%v", removed, err)
+ }
+ if _, _, err := f.store.Get(f.ctx, r.Holder.Key); err != nil {
+ t.Fatal("replacement lost")
+ }
+}
+
+func TestPlacementStartKeepsInstallationAndExcludesDelete(t *testing.T) {
+ f := newFixture(t)
+ obs := f.serving(t, "shop.bab.example.com")
+ stamp := obs.Claim.Holder.Stamp
+ placed, err := hostclaim.Update(f.ctx, f.store, obs, BeginPlacement("place", "test", "fleet/customer", f.now, []string{"ie", "us"}, []string{"box-a", "box-b"}))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if placed.Claim.Op.Exec != obs.Claim.Op.Exec || placed.Claim.Holder.Stamp != stamp || placed.Claim.Op.Intent["target"] != "fleet/customer" {
+ t.Fatal("placement changed authority")
+ }
+ rec, err := ReadRecord(f.ctx, f.store, obs.Claim.Holder.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := hostclaim.Update(f.ctx, f.store, placed, hostclaim.BeginDelete("delete", "test", rec.ETag, f.now, nil, nil)); err == nil {
+ t.Fatal("delete overlapped placement")
+ }
+}
+
+func TestFenceReleasePreservesHistoryAndRefusesAnotherPurge(t *testing.T) {
+ f := newFixture(t)
+ before, _ := hostclaim.Read(f.ctx, f.store, "shop.bab.example.com")
+ obs, err := hostclaim.Update(f.ctx, f.store, before, func(c *hostclaim.Claim) error {
+ c.Lost = []string{"lost-box"}
+ c.QuietAfter = map[string]time.Time{"ie": f.now.Add(time.Hour)}
+ return FencePurge("purge", f.now)(c)
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := hostclaim.Update(f.ctx, f.store, obs, ReleasePurgeFence("other")); err == nil {
+ t.Fatal("released other's fence")
+ }
+ obs, err = hostclaim.Update(f.ctx, f.store, obs, ReleasePurgeFence("purge"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !obs.Claim.Free() || len(obs.Claim.Lost) != 1 || obs.Claim.QuietAfter["ie"].IsZero() {
+ t.Fatal("lost history")
+ }
+}
+
+type repairAuthority struct {
+ Authority
+ afterAdoption func()
+}
+
+func (a repairAuthority) Adoption(ctx context.Context, r Record) (string, string, []string, []string, error) {
+ region, box, regions, boxes, err := a.Authority.Adoption(ctx, r)
+ if a.afterAdoption != nil {
+ a.afterAdoption()
+ }
+ return region, box, regions, boxes, err
+}
+
+func TestAdoptLegacyRepairRequiresCensusIdentityAndFreshRecord(t *testing.T) {
+ for _, scenario := range []string{"wrong-executor", "not-serving", "owner-not-touched", "changed-record", "missing-record", "record-read-error"} {
+ t.Run(scenario, func(t *testing.T) {
+ f := newFixture(t)
+ before := f.serving(t, "legacy.example.com")
+ var err error
+ before, err = hostclaim.Update(f.ctx, f.store, before, func(c *hostclaim.Claim) error {
+ switch scenario {
+ case "wrong-executor":
+ c.Op.Exec = "box-b"
+ c.Touched.Boxes = append(c.Touched.Boxes, "box-b")
+ case "not-serving":
+ c.Op.Kind, c.Op.ID = hostclaim.KindMoving, "move"
+ case "owner-not-touched":
+ c.Touched.Boxes = []string{"box-b"}
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ r, err := ReadRecord(f.ctx, f.store, before.Claim.Holder.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var body provision.Record
+ if err := json.Unmarshal(r.Body, &body); err != nil {
+ t.Fatal(err)
+ }
+ body.Acquisition = "ownership-acquisition"
+ raw, err := json.Marshal(body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := f.store.Put(f.ctx, r.Holder.Key, raw, r.ETag); err != nil {
+ t.Fatal(err)
+ }
+ r, err = ReadRecord(f.ctx, f.store, r.Holder.Key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ want := ErrIdentity
+ if scenario == "changed-record" {
+ want = ErrPending
+ } else if scenario == "missing-record" {
+ want = os.ErrNotExist
+ }
+ f.c.Authority = repairAuthority{Authority: f.auth, afterAdoption: func() {
+ switch scenario {
+ case "changed-record":
+ if _, err := f.store.Put(f.ctx, r.Holder.Key, append(raw, ' '), r.ETag); err != nil {
+ t.Fatal(err)
+ }
+ case "missing-record":
+ if err := f.store.DeleteIfMatch(f.ctx, r.Holder.Key, r.ETag); err != nil {
+ t.Fatal(err)
+ }
+ case "record-read-error":
+ f.store.failGet = r.Holder.Key
+ }
+ }}
+ writes := 0
+ f.store.beforePut = func(string) error { writes++; return nil }
+ _, err = f.c.Adopt(f.ctx, r)
+ if scenario == "record-read-error" {
+ if err == nil || err.Error() != "read failed" {
+ t.Errorf("repair error = %v, want record read failure", err)
+ }
+ } else if !errors.Is(err, want) {
+ t.Errorf("repair error = %v, want %v", err, want)
+ }
+ if scenario == "changed-record" {
+ writes--
+ }
+ after, readErr := hostclaim.Read(f.ctx, f.store, before.Host)
+ if readErr != nil || writes != 0 || after.ETag != before.ETag || !reflect.DeepEqual(after.Claim, before.Claim) {
+ t.Fatalf("refused repair changed claim: writes=%d, read=%v", writes, readErr)
+ }
+ })
+ }
+}
diff --git a/internal/hostops/purge.go b/internal/hostops/purge.go
new file mode 100644
index 000000000..ba2ae03d5
--- /dev/null
+++ b/internal/hostops/purge.go
@@ -0,0 +1,196 @@
+package hostops
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "os"
+ "time"
+
+ "github.com/carlosframework/platform/internal/controlname"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type Purge struct {
+ ID string `json:"id"`
+ Phase string `json:"phase"`
+}
+
+var phases = []string{"reserved", "adopted", "claimed", "torn-down", "retired", "done"}
+
+func phaseIndex(s string) int {
+ for i, p := range phases {
+ if s == p {
+ return i
+ }
+ }
+ return -1
+}
+func PurgeKey(acct, app string) (string, error) {
+ if !controlname.ValidAccount(acct) || !controlname.ValidApp(app) {
+ return "", ErrIdentity
+ }
+ return "apps/" + acct + "/" + app + "/purge.json", nil
+}
+func ReadPurge(ctx context.Context, s release.Store, acct, app string) (Purge, string, error) {
+ key, err := PurgeKey(acct, app)
+ if err != nil {
+ return Purge{}, "", err
+ }
+ body, etag, err := s.Get(ctx, key)
+ if err != nil {
+ return Purge{}, "", err
+ }
+ var p Purge
+ if err := decode(body, &p); err != nil {
+ return Purge{}, "", err
+ }
+ if p.ID == "" || phaseIndex(p.Phase) < 0 || etag == "" {
+ return Purge{}, "", ErrIdentity
+ }
+ return p, etag, nil
+}
+func ReservePurge(ctx context.Context, s release.Store, acct, app, id string) (Purge, string, error) {
+ key, err := PurgeKey(acct, app)
+ if err != nil || id == "" {
+ return Purge{}, "", ErrIdentity
+ }
+ p := Purge{ID: id, Phase: "reserved"}
+ body, err := json.Marshal(p)
+ if err != nil {
+ return Purge{}, "", err
+ }
+ etag, err := s.Put(ctx, key, body, "")
+ if errors.Is(err, release.ErrCASConflict) {
+ existing, tag, rerr := ReadPurge(ctx, s, acct, app)
+ if rerr != nil {
+ return Purge{}, "", rerr
+ }
+ if existing.ID != id {
+ return existing, tag, ErrPending
+ }
+ return existing, tag, nil
+ }
+ return p, etag, err
+}
+
+// AdvancePurge preserves additional journal fields owned by the final durable
+// host-set schema. Preconditions for the next phase belong to the coordinator.
+func AdvancePurge(ctx context.Context, s release.Store, acct, app, id, from, to string) error {
+ if phaseIndex(from) < 0 || phaseIndex(to) != phaseIndex(from)+1 {
+ return ErrIdentity
+ }
+ key, err := PurgeKey(acct, app)
+ if err != nil {
+ return err
+ }
+ body, etag, err := s.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ var p Purge
+ if err := decode(body, &p); err != nil {
+ return err
+ }
+ if p.ID != id || p.Phase != from || etag == "" {
+ return ErrPending
+ }
+ var fields map[string]json.RawMessage
+ if err := decode(body, &fields); err != nil {
+ return err
+ }
+ fields["phase"], err = json.Marshal(to)
+ if err != nil {
+ return err
+ }
+ body, err = json.Marshal(fields)
+ if err != nil {
+ return err
+ }
+ _, err = s.Put(ctx, key, body, etag)
+ return err
+}
+
+// Only the completed coordinator may call this, after all fence-release writes.
+func RemovePurge(ctx context.Context, s release.Store, acct, app, id string) error {
+ p, etag, err := ReadPurge(ctx, s, acct, app)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ if p.ID != id || p.Phase != "done" {
+ return ErrPending
+ }
+ key, err := PurgeKey(acct, app)
+ if err != nil {
+ return err
+ }
+ return release.DeleteIfMatch(ctx, s, key, etag)
+}
+
+func FencePurge(id string, now time.Time) func(*hostclaim.Claim) error {
+ return func(c *hostclaim.Claim) error {
+ if id == "" || now.IsZero() {
+ return ErrIdentity
+ }
+ if !c.Free() || len(c.Obligations) != 0 {
+ return ErrPending
+ }
+ c.Op = hostclaim.Op{Kind: hostclaim.KindPurging, ID: id, StartedAt: now.UTC()}
+ return nil
+ }
+}
+func ReleasePurgeFence(id string) func(*hostclaim.Claim) error {
+ return func(c *hostclaim.Claim) error {
+ if id == "" || c.Holder != nil || c.Op.Kind != hostclaim.KindPurging || c.Op.ID != id || len(c.Obligations) != 0 {
+ return ErrPending
+ }
+ c.Op = hostclaim.Op{}
+ return nil
+ }
+}
+func BeginPurge(id, by string, now time.Time, regions, boxes []string) func(*hostclaim.Claim) error {
+ return func(c *hostclaim.Claim) error {
+ if id == "" || now.IsZero() {
+ return ErrIdentity
+ }
+ if err := hostclaim.StartGuard(c, regions, boxes); err != nil {
+ return err
+ }
+ c.Op = hostclaim.Op{Kind: hostclaim.KindPurging, ID: id, By: by, StartedAt: now.UTC(), Exec: c.Op.Exec, Participants: c.Op.Participants}
+ return nil
+ }
+}
+func PurgeRecordIntent(id, key, etag string) func(*hostclaim.Claim) error {
+ return func(c *hostclaim.Claim) error {
+ if id == "" || etag == "" || c.Op.Kind != hostclaim.KindPurging || c.Op.ID != id || c.Holder == nil || c.Holder.Key != key {
+ return ErrIdentity
+ }
+ if c.Op.Intent == nil {
+ c.Op.Intent = map[string]string{}
+ }
+ if old := c.Op.Intent["etag"]; old != "" && old != etag {
+ return ErrPending
+ }
+ c.Op.Intent["key"], c.Op.Intent["etag"] = key, etag
+ return nil
+ }
+}
+
+// No completion transition is supplied until placement's executor/evidence
+// contract is resolved. This start transition alone grants no new execution.
+func BeginPlacement(id, by, target string, now time.Time, regions, boxes []string) func(*hostclaim.Claim) error {
+ return func(c *hostclaim.Claim) error {
+ if id == "" || now.IsZero() {
+ return ErrIdentity
+ }
+ if err := hostclaim.StartGuard(c, regions, boxes); err != nil {
+ return err
+ }
+ c.Op = hostclaim.Op{Kind: hostclaim.KindPlacing, ID: id, By: by, StartedAt: now.UTC(), Exec: c.Op.Exec, Participants: c.Op.Participants, Intent: map[string]string{"target": target}}
+ return nil
+ }
+}
diff --git a/internal/hostops/record.go b/internal/hostops/record.go
new file mode 100644
index 000000000..25223712f
--- /dev/null
+++ b/internal/hostops/record.go
@@ -0,0 +1,182 @@
+package hostops
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/controlname"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+var (
+ ErrPending = errors.New("hostops: pending")
+ ErrContract = errors.New("hostops: required contract unavailable")
+ ErrIdentity = errors.New("hostops: identity mismatch")
+)
+
+type Record struct {
+ Holder hostclaim.NewHolder
+ Body []byte
+ ETag string
+}
+
+func decode(body []byte, out any) error {
+ d := json.NewDecoder(bytes.NewReader(body))
+ if err := d.Decode(out); err != nil {
+ return err
+ }
+ if _, err := d.Token(); err != io.EOF {
+ return ErrIdentity
+ }
+ return nil
+}
+
+func ReadRecord(ctx context.Context, s release.Store, key string) (Record, error) {
+ parts := strings.Split(key, "/")
+ if len(parts) != 5 || parts[0] != "apps" || !controlname.ValidAccount(parts[1]) || !controlname.ValidApp(parts[2]) || !strings.HasSuffix(parts[4], ".json") {
+ return Record{}, ErrIdentity
+ }
+ body, etag, err := s.Get(ctx, key)
+ if err != nil {
+ return Record{}, err
+ }
+ if etag == "" {
+ return Record{}, ErrIdentity
+ }
+ var binding struct {
+ Acquisition string `json:"acq"`
+ }
+ if err := decode(body, &binding); err != nil {
+ return Record{}, err
+ }
+ n := hostclaim.NewHolder{Acquisition: binding.Acquisition, Key: key, Account: parts[1], App: parts[2]}
+ switch parts[3] {
+ case "instances":
+ var r provision.Record
+ if err := decode(body, &r); err != nil {
+ return Record{}, err
+ }
+ if r.Account != n.Account || r.App != n.App || key != provision.RecordKey(r.Account, r.App, r.Host) || !controlname.ValidHostShape(r.Host) {
+ return Record{}, ErrIdentity
+ }
+ // DecodeRecord is also the ordinary instance reader's shape validator.
+ if _, err := provision.DecodeRecord(body); err != nil {
+ return Record{}, err
+ }
+ n.Family, n.Host, n.RecordRegion, n.CreatedAt = "instance", r.Host, r.Region, r.CreatedAt
+ case "sidecars":
+ var r provision.SidecarRecord
+ if err := decode(body, &r); err != nil {
+ return Record{}, err
+ }
+ if r.Account != n.Account || r.App != n.App || !controlname.ValidLabel(r.Name) || key != provision.SidecarRecordKey(r.Account, r.App, r.Name) || r.Host != provision.SidecarHost(r.Name, r.App, r.Account) {
+ return Record{}, ErrIdentity
+ }
+ n.Family, n.Host, n.Name, n.RecordRegion, n.CreatedAt = "sidecar", r.Host, r.Name, r.Region, r.CreatedAt
+ default:
+ return Record{}, ErrIdentity
+ }
+ return Record{Holder: n, Body: body, ETag: etag}, nil
+}
+
+func (r Record) Incarnation() (string, error) {
+ n := r.Holder
+ return hostclaim.Incarnation(n.Account, n.App, n.Host, n.CreatedAt)
+}
+
+func matches(r Record, h *hostclaim.Holder) bool {
+ if h == nil {
+ return false
+ }
+ n := r.Holder
+ inc, err := r.Incarnation()
+ return err == nil && n.Acquisition == h.Acquisition && n.Key == h.Key && n.Family == h.Family && n.Account == h.Account && n.App == h.App && n.Name == h.Name && n.CreatedAt.Equal(h.CreatedAt) && inc == h.Incarnation
+}
+
+// The caller reads the record first. This function performs the later claim read.
+func RemoveOrphan(ctx context.Context, s release.Store, r Record) (bool, error) {
+ if err := CheckHostQuarantine(ctx, s, r.Holder.Host); err != nil {
+ return false, err
+ }
+ inc, err := r.Incarnation()
+ if err != nil {
+ return false, err
+ }
+ obs, err := hostclaim.Read(ctx, s, r.Holder.Host)
+ if err != nil {
+ return false, err
+ }
+ if !hostclaim.IsOrphan(inc, obs) {
+ return false, nil
+ }
+ if err := release.DeleteIfMatch(ctx, s, r.Holder.Key, r.ETag); err != nil {
+ return false, err
+ }
+ return true, nil
+}
+
+func Records(ctx context.Context, s release.Store, acct, app string) ([]Record, error) {
+ if !controlname.ValidAccount(acct) || !controlname.ValidApp(app) {
+ return nil, ErrIdentity
+ }
+ var out []Record
+ for _, prefix := range []string{provision.RecordsPrefix(acct, app), provision.SidecarsPrefix(acct, app)} {
+ keys, err := s.List(ctx, prefix)
+ if err != nil {
+ return nil, err
+ }
+ for _, key := range keys {
+ tail, ok := strings.CutPrefix(key, prefix)
+ if !ok {
+ return nil, ErrIdentity
+ }
+ if strings.Contains(tail, "/") || !strings.HasSuffix(tail, ".json") {
+ continue
+ }
+ r, err := ReadRecord(ctx, s, key)
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ if err != nil {
+ return nil, err
+ }
+ out = append(out, r)
+ }
+ }
+ return out, nil
+}
+
+// PrepareInstance accepts only an unstamped internal request, never a caller time.
+func PrepareInstance(r provision.Record, now time.Time, region string) (Record, error) {
+ if !r.CreatedAt.IsZero() || now.IsZero() || region == "" || len(r.Maintenance) != 0 {
+ return Record{}, ErrIdentity
+ }
+ r.CreatedAt = now.UTC()
+ body, err := json.Marshal(r)
+ if err != nil {
+ return Record{}, err
+ }
+ n := hostclaim.NewHolder{Acquisition: r.Acquisition, Family: "instance", Key: provision.RecordKey(r.Account, r.App, r.Host), Account: r.Account, App: r.App, Host: r.Host, RecordRegion: r.Region, Region: region, CreatedAt: r.CreatedAt}
+ return Record{Holder: n, Body: body}, nil
+}
+
+func PrepareSidecar(r provision.SidecarRecord, now time.Time, region string) (Record, error) {
+ if !r.CreatedAt.IsZero() || now.IsZero() || region == "" {
+ return Record{}, ErrIdentity
+ }
+ r.CreatedAt = now.UTC()
+ body, err := json.Marshal(r)
+ if err != nil {
+ return Record{}, err
+ }
+ n := hostclaim.NewHolder{Acquisition: r.Acquisition, Family: "sidecar", Key: provision.SidecarRecordKey(r.Account, r.App, r.Name), Account: r.Account, App: r.App, Name: r.Name, Host: r.Host, RecordRegion: r.Region, Region: region, CreatedAt: r.CreatedAt}
+ return Record{Holder: n, Body: body}, nil
+}
diff --git a/internal/hostops/recovery.go b/internal/hostops/recovery.go
new file mode 100644
index 000000000..26a9f2929
--- /dev/null
+++ b/internal/hostops/recovery.go
@@ -0,0 +1,216 @@
+package hostops
+
+import (
+ "context"
+ "errors"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// ListClaims is discovery only. Every action subsequently reads its own key.
+func ListClaims(ctx context.Context, s release.Store) ([]string, error) {
+ keys, err := s.List(ctx, "control/hosts/")
+ if err != nil {
+ return nil, err
+ }
+ var hosts []string
+ for _, key := range keys {
+ tail, ok := strings.CutPrefix(key, "control/hosts/")
+ if !ok {
+ return nil, ErrIdentity
+ }
+ if strings.Contains(tail, "/") || !strings.HasSuffix(tail, ".json") {
+ continue
+ }
+ host := strings.TrimSuffix(tail, ".json")
+ if _, err := hostclaim.Key(host); err != nil {
+ return nil, err
+ }
+ hosts = append(hosts, host)
+ }
+ return hosts, nil
+}
+
+type Recovery struct {
+ SweepAdmissions func(context.Context) error
+ Controller Controller
+ // These are required integrations, not no-op defaults.
+ Purge func(context.Context, hostclaim.Observed) error
+ Placement func(context.Context, hostclaim.Observed) error
+ SweepRecords func(context.Context) error
+ SweepPurges func(context.Context) error
+}
+
+func (r Recovery) Pass(ctx context.Context) error {
+ c := r.Controller
+ if err := c.ready(); err != nil {
+ return err
+ }
+ if r.Purge == nil || r.Placement == nil || r.SweepRecords == nil || r.SweepPurges == nil || r.SweepAdmissions == nil {
+ return ErrContract
+ }
+ hosts, err := ListClaims(ctx, c.Store)
+ if err != nil {
+ return err
+ }
+ var all []error
+ for _, host := range hosts {
+ if err := ctx.Err(); err != nil {
+ return errors.Join(append(all, err)...)
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, host)
+ if err != nil {
+ all = append(all, err)
+ continue
+ }
+ if obs.Absent {
+ continue
+ }
+ op := obs.Claim.Op
+ // Zero time is unknown age, never old enough. Freeing empty parking is safe
+ // independently of age; this includes Plan 4's holderless sweep retries.
+ if op.Kind == hostclaim.KindParking && len(obs.Claim.Obligations) == 0 {
+ err = c.Free(ctx, host, op.ID)
+ } else if op.Kind != "" && op.Kind != hostclaim.KindServing && !op.StartedAt.IsZero() && c.Now().After(op.StartedAt.Add(5*time.Minute)) {
+ switch op.Kind {
+ case hostclaim.KindCreating:
+ err = c.RecoverCreate(ctx, host)
+ case hostclaim.KindDeleting:
+ err = c.Retire(ctx, host, op.ID)
+ case hostclaim.KindPurging:
+ err = r.Purge(ctx, obs)
+ case hostclaim.KindPlacing:
+ err = r.Placement(ctx, obs)
+ }
+ }
+ if err != nil && !errors.Is(err, ErrPending) {
+ all = append(all, err)
+ }
+ }
+ // Records have their own uncapped scan: late orphans outlive the one-hour
+ // cancellation hint; records without claims need the shared adoption path.
+ if err := r.SweepRecords(ctx); err != nil {
+ all = append(all, err)
+ }
+ // Reservations can exist without any claim (crash just after reserve).
+ if err := r.SweepPurges(ctx); err != nil {
+ all = append(all, err)
+ }
+ if err := r.SweepAdmissions(ctx); err != nil {
+ all = append(all, err)
+ }
+ return errors.Join(all...)
+}
+
+type App struct{ Account, Name string }
+
+func Apps(ctx context.Context, s release.Store) ([]App, error) {
+ accounts, err := s.ListDirs(ctx, "apps/")
+ if err != nil {
+ return nil, err
+ }
+ var out []App
+ for _, prefix := range accounts {
+ account := strings.TrimSuffix(strings.TrimPrefix(prefix, "apps/"), "/")
+ if prefix != "apps/"+account+"/" {
+ return nil, ErrIdentity
+ }
+ // The retained legacy layout is apps/<app>/..., so its first segment is
+ // an app name, not necessarily an account. Skip names that are valid apps
+ // but invalid accounts before treating this as a qualified account root.
+ // A letters-only legacy name can pass both validators, so preserve its
+ // children as candidates and let the exact record and purge parsers
+ // validate them. The path alone cannot tell that case from a real account
+ // containing apps with those legal names; never infer deletion authority
+ // over retained legacy content from this prefix shape.
+ if !release.ValidAccount(account) {
+ if release.ValidApp(account) && prefix == release.LegacyAppPrefix(account) {
+ continue
+ }
+ return nil, ErrIdentity
+ }
+ apps, err := s.ListDirs(ctx, prefix)
+ if err != nil {
+ return nil, err
+ }
+ for _, ap := range apps {
+ app := strings.TrimSuffix(strings.TrimPrefix(ap, prefix), "/")
+ if _, err := PurgeKey(account, app); err != nil || ap != prefix+app+"/" {
+ return nil, ErrIdentity
+ }
+ out = append(out, App{Account: account, Name: app})
+ }
+ }
+ return out, nil
+}
+
+func (c Controller) SweepAllRecords(ctx context.Context) error {
+ if err := c.ready(); err != nil {
+ return err
+ }
+ apps, err := Apps(ctx, c.Store)
+ if err != nil {
+ return err
+ }
+ var all []error
+ for _, app := range apps {
+ records, err := Records(ctx, c.Store, app.Account, app.Name)
+ if err != nil {
+ all = append(all, err)
+ continue
+ }
+ for _, listed := range records {
+ record, err := ReadRecord(ctx, c.Store, listed.Holder.Key)
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ if err != nil {
+ all = append(all, err)
+ continue
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, record.Holder.Host)
+ if err != nil {
+ all = append(all, err)
+ continue
+ }
+ if obs.Absent {
+ err = ErrPending
+ } else {
+ _, err = RemoveOrphan(ctx, c.Store, record)
+ }
+ if err != nil {
+ all = append(all, err)
+ }
+ }
+ }
+ return errors.Join(all...)
+}
+
+func SweepAllPurges(ctx context.Context, s release.Store, replay func(context.Context, App, Purge) error) error {
+ if replay == nil {
+ return ErrContract
+ }
+ apps, err := Apps(ctx, s)
+ if err != nil {
+ return err
+ }
+ var all []error
+ for _, app := range apps {
+ p, _, err := ReadPurge(ctx, s, app.Account, app.Name)
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ if err != nil {
+ all = append(all, err)
+ continue
+ }
+ if err := replay(ctx, app, p); err != nil {
+ all = append(all, err)
+ }
+ }
+ return errors.Join(all...)
+}
diff --git a/internal/hostops/remote.go b/internal/hostops/remote.go
new file mode 100644
index 000000000..e1c44b737
--- /dev/null
+++ b/internal/hostops/remote.go
@@ -0,0 +1,47 @@
+package hostops
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/park"
+)
+
+type ParkScopeProvider func(context.Context, hostclaim.Observed, hostclaim.Obligation) (hostclaim.ParkScope, error)
+
+// RunOwnedRemote is called only by the appointed regional worker. It never
+// takes another region's chore or acknowledges local work on another box.
+func RunOwnedRemote(ctx context.Context, e *park.Engine, w park.Work, scope ParkScopeProvider) error {
+ if e == nil || e.Claims == nil || e.Region == "" || scope == nil || e.Authorize == nil || e.View == nil || e.Leases == nil {
+ return ErrContract
+ }
+ obs, err := hostclaim.Read(ctx, e.Claims, w.Host)
+ if err != nil {
+ return err
+ }
+ if obs.Absent {
+ return ErrPending
+ }
+ for _, ob := range obs.Claim.Obligations {
+ if ob.ID != w.ID || ob.Op != w.Op {
+ continue
+ }
+ if ob.Region != e.Region || ob.Owner != "region:"+e.Region || (ob.Kind != "park-replica" && ob.Kind != "park-generation") {
+ return ErrIdentity
+ }
+ if err := e.Authorize(ctx, obs, ob); err != nil {
+ return err
+ }
+ if ob.Park == nil {
+ p, err := scope(ctx, obs, ob)
+ if err != nil {
+ return err
+ }
+ if err := e.Bind(ctx, w, p); err != nil {
+ return err
+ }
+ }
+ return e.Run(ctx, w)
+ }
+ return ErrPending
+}
diff --git a/internal/hostops/retirement.go b/internal/hostops/retirement.go
new file mode 100644
index 000000000..f0ee2979d
--- /dev/null
+++ b/internal/hostops/retirement.go
@@ -0,0 +1,69 @@
+package hostops
+
+import (
+ "context"
+ "errors"
+ "os"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+func (c Controller) RetireRecord(ctx context.Context, ref provision.RetireRef) error {
+ if err := c.ready(); err != nil {
+ return err
+ }
+ key := provision.RecordKey(ref.Account, ref.App, ref.Host)
+ if ref.Family == "sidecar" {
+ key = provision.SidecarRecordKey(ref.Account, ref.App, ref.Name)
+ }
+ rec, rerr := ReadRecord(ctx, c.Store, key)
+ if rerr != nil && !errors.Is(rerr, os.ErrNotExist) {
+ return rerr
+ }
+ obs, err := hostclaim.Read(ctx, c.Store, ref.Host)
+ if err != nil {
+ return err
+ }
+ if obs.Absent {
+ if errors.Is(rerr, os.ErrNotExist) {
+ return nil
+ }
+ return ErrPending
+ }
+ h := obs.Claim.Holder
+ if h == nil {
+ if rerr == nil {
+ _, err = RemoveOrphan(ctx, c.Store, rec)
+ return err
+ }
+ if obs.Claim.Free() {
+ return nil
+ }
+ return ErrPending
+ }
+ if h.Key != key || h.Family != ref.Family || h.Account != ref.Account || h.App != ref.App || (ref.Incarnation != "" && ref.Incarnation != h.Incarnation) {
+ return ErrIdentity
+ }
+ if rerr == nil && !matches(rec, h) {
+ return ErrIdentity
+ }
+ switch obs.Claim.Op.Kind {
+ case hostclaim.KindServing:
+ if rerr != nil {
+ return ErrPending
+ } // Old-console deletion needs operator recovery.
+ obs, err = c.BeginRetire(ctx, key, ref.By, h.Incarnation)
+ if err != nil {
+ return err
+ }
+ return c.Retire(ctx, ref.Host, obs.Claim.Op.ID)
+ case hostclaim.KindDeleting:
+ return c.Retire(ctx, ref.Host, obs.Claim.Op.ID)
+ case hostclaim.KindParking:
+ // Record withdrawal completed. Chores still block reuse in the claim.
+ return nil
+ default:
+ return ErrPending // A purge is driven by its journal, never this path.
+ }
+}
diff --git a/internal/provision/claim_export.go b/internal/provision/claim_export.go
new file mode 100644
index 000000000..3a41dc9e6
--- /dev/null
+++ b/internal/provision/claim_export.go
@@ -0,0 +1,3 @@
+package provision
+
+func ClaimEverywhere(rec Record) bool { return everywhereClass(rec) }
diff --git a/internal/provision/provision.go b/internal/provision/provision.go
index a20198f96..37ac2e594 100644
--- a/internal/provision/provision.go
+++ b/internal/provision/provision.go
@@ -37,6 +37,7 @@ import (
// Account vs the key's segments); GetRecord and ListRecords enforce that, a
// record that lies about identity is never guessed at (design §3.2).
type Record struct {
+ Acquisition string `json:"acq,omitempty"`
// Maintenance is preserved by ordinary writers; only a future guarded
// maintenance transition may install, replace, or clear it.
Maintenance MaintenanceProjection `json:"maintenance,omitempty"`
diff --git a/internal/provision/retirement.go b/internal/provision/retirement.go
new file mode 100644
index 000000000..aab7a2564
--- /dev/null
+++ b/internal/provision/retirement.go
@@ -0,0 +1,53 @@
+package provision
+
+import (
+ "context"
+ "errors"
+
+ "github.com/carlosframework/platform/internal/controlname"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type RetireRef struct {
+ Family, Account, App, Host, Name, By string
+ // Required by rollbacks: never retire a successor to the failed attempt.
+ Incarnation string
+}
+type ClaimRetirer interface {
+ RetireRecord(context.Context, RetireRef) error
+}
+type RetireOptions struct {
+ HostClaim bool
+ Claims ClaimRetirer
+}
+
+var ErrClaimRetirer = errors.New("provision: claim retirement unavailable")
+
+func RetireRecord(ctx context.Context, s release.Store, ref RetireRef, opt RetireOptions) error {
+ if !opt.HostClaim {
+ switch ref.Family {
+ case "instance":
+ return DeleteRecord(ctx, s, ref.Account, ref.App, ref.Host)
+ case "sidecar":
+ return DeleteSidecarRecord(ctx, s, ref.Account, ref.App, ref.Name)
+ default:
+ return ErrClaimRetirer
+ }
+ }
+ if opt.Claims == nil || !controlname.ValidAccount(ref.Account) || !controlname.ValidApp(ref.App) || !controlname.ValidHostShape(ref.Host) {
+ return ErrClaimRetirer
+ }
+ switch ref.Family {
+ case "instance":
+ if ref.Name != "" {
+ return ErrClaimRetirer
+ }
+ case "sidecar":
+ if !controlname.ValidLabel(ref.Name) || ref.Host != SidecarHost(ref.Name, ref.App, ref.Account) {
+ return ErrClaimRetirer
+ }
+ default:
+ return ErrClaimRetirer
+ }
+ return opt.Claims.RetireRecord(ctx, ref)
+}
diff --git a/internal/provision/retirement_test.go b/internal/provision/retirement_test.go
new file mode 100644
index 000000000..e0a4f45f4
--- /dev/null
+++ b/internal/provision/retirement_test.go
@@ -0,0 +1,41 @@
+package provision
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type retireSpy struct{ calls int }
+
+func (s *retireSpy) RetireRecord(context.Context, RetireRef) error {
+ s.calls++
+ return errors.New("claim path")
+}
+func TestRetireRecordFlagOffAndRequiredProvider(t *testing.T) {
+ ctx := context.Background()
+ store := release.NewDirStore(t.TempDir())
+ spy := &retireSpy{}
+ for _, family := range []string{"instance", "sidecar"} {
+ ref := RetireRef{Family: family, Account: "bab", App: "jam", Host: "shop.bab.example.com"}
+ if family == "sidecar" {
+ ref.Name = "job"
+ ref.Host = SidecarHost(ref.Name, ref.App, ref.Account)
+ }
+ if err := RetireRecord(ctx, store, ref, RetireOptions{Claims: spy}); err != nil {
+ t.Fatal(err)
+ }
+ if spy.calls != 0 {
+ t.Fatal("flag off called claims")
+ }
+ if err := RetireRecord(ctx, store, ref, RetireOptions{HostClaim: true}); !errors.Is(err, ErrClaimRetirer) {
+ t.Fatal(err)
+ }
+ }
+ ref := RetireRef{Family: "instance", Account: "bab", App: "jam", Host: "shop.bab.example.com"}
+ if err := RetireRecord(ctx, store, ref, RetireOptions{HostClaim: true, Claims: spy}); err == nil || spy.calls != 1 {
+ t.Fatal("claim path not required")
+ }
+}
diff --git a/internal/provision/sidecar.go b/internal/provision/sidecar.go
index d6891ea85..f49fbaa4f 100644
--- a/internal/provision/sidecar.go
+++ b/internal/provision/sidecar.go
@@ -69,6 +69,7 @@ type SidecarManifest struct {
// identity is never guessed at, the same discipline GetRecord applies to
// instances.
type SidecarRecord struct {
+ Acquisition string `json:"acq,omitempty"`
Schema int `json:"schema"` // 1
Name string `json:"name"`
Host string `json:"host"`