amadan
hostops, hostclaim, hostfence, console, host: frozen-backfill adoption, the recovery entry point, fence verification and the pre-flip relocating conversion (claim Plan 5 rev 2 Tasks 5-6, C5/C7)
Inert with CARLOS_HOST_CLAIM unset. Adoption only through the frozen #232 backfill reconcile; the recovery pass never adopts. ConvertMovingToRelocating holds at relocating/pinning, with no activation.
Co-Authored-By: Codex <noreply@openai.com> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
27 files changed,
+1776
−38
cmd/console/main.go+4 −0internal/console/hostclaimbackfill.go+82 −0internal/console/hostclaimops.go+2 −2internal/domains/claims.go+10 −7internal/host/agent.go+10 −3internal/host/claimbackfill.go+32 −0internal/hostactors/censuskey.go+59 −0internal/hostactors/roster.go+15 −11internal/hostclaim/adopt.go+1 −1internal/hostclaim/claim.go+5 −1internal/hostclaim/create.go+2 −1internal/hostclaim/fence.go+109 −0internal/hostclaim/relocating.go+142 −0internal/hostclaim/relocating_test.go+251 −0internal/hostfence/claim_fence.go+38 −0internal/hostfence/claim_fence_test.go+72 −0internal/hostops/backfill.go+380 −0internal/hostops/backfill_controller.go+119 −0internal/hostops/backfill_publish.go+162 −0internal/hostops/backfill_test.go+233 −0internal/hostops/operations.go+28 −3internal/hostops/record.go+13 −4internal/hostops/recovery.go+1 −1internal/hostops/retirement.go+1 −4internal/provision/claim_export.go+3 −0internal/provision/provision.go+1 −0internal/provision/sidecar.go+1 −0
CI passed — run details
diff --git a/cmd/console/main.go b/cmd/console/main.go| index c70e9ffda..dc28c9523 100644 |
| --- a/cmd/console/main.go |
| +++ b/cmd/console/main.go |
| @@ -392,6 +392,10 @@ func serve() { |
| // objects and pulls the AWS bill (internal/console/economicspass.go). |
| // Same placement rule as the alert watcher: started here, never in |
| // NewServer, so constructing a Server stays goroutine-free. |
| + if err := srv.ConfigureClaimBackfill(); err != nil { |
| + log.Fatalf("console: claim recovery: %v", err) |
| + } |
| + go srv.StartClaimRecoveryPass(context.Background()) |
| go srv.StartEconomicsPass(context.Background()) |
| // The infrastructure inventory pass writes infra/inventory.json for |
| // /fleet/architecture (internal/console/infrapass.go). Same placement |
diff --git a/internal/console/hostclaimbackfill.go b/internal/console/hostclaimbackfill.go| new file mode 100644 |
| index 000000000..f4936d0c4 |
| --- /dev/null |
| +++ b/internal/console/hostclaimbackfill.go |
| @@ -0,0 +1,82 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "crypto/rand" |
| + "encoding/hex" |
| + "errors" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostops" |
| +) |
| + |
| +func claimOperationID() string { |
| + var b [16]byte |
| + if _, err := rand.Read(b[:]); err != nil { |
| + return "" |
| + } |
| + return hex.EncodeToString(b[:]) |
| +} |
| + |
| +// Called after NewServer, before background workers. It starts no goroutine. |
| +func (s *Server) ConfigureClaimBackfill() error { |
| + if !s.cfg.HostClaim { |
| + return nil |
| + } |
| + if s.cfg.Releases == nil || s.cfg.Bucket == "" || len(s.cfg.ReleasePub) == 0 { |
| + return hostops.ErrContract |
| + } |
| + if s.cfg.HostClaimOps == nil { |
| + d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: s.cfg.ReleasePub, Bucket: s.cfg.Bucket} |
| + c := hostops.NewBackfillController(s.cfg.Releases, d, time.Now, claimOperationID) |
| + s.cfg.HostClaimOps = &c |
| + } |
| + return nil |
| +} |
| +func (s *Server) ReconcileHostClaims(ctx context.Context) error { |
| + if !s.cfg.HostClaim { |
| + return nil |
| + } |
| + if s.cfg.HostClaimOps == nil { |
| + return hostops.ErrContract |
| + } |
| + return hostops.ReconcileClaims(ctx, *s.cfg.HostClaimOps, s.adoptHostRecord) |
| +} |
| + |
| +// Exported for the moves console pass; construction and startup remain separate. |
| +func (s *Server) RunClaimRecoveryPass(ctx context.Context) error { |
| + if !s.cfg.HostClaim { |
| + return nil |
| + } |
| + var errs []error |
| + if err := s.ReconcileHostClaims(ctx); err != nil { |
| + errs = append(errs, err) |
| + } |
| + if s.cfg.HostClaimRecovery != nil { |
| + if err := s.cfg.HostClaimRecovery.Pass(ctx); err != nil { |
| + errs = append(errs, err) |
| + } |
| + } |
| + return errors.Join(errs...) |
| +} |
| + |
| +// A door may only consume a claim after the backfill has established it. |
| +func claimedRecord(ctx context.Context, c *hostops.Controller, key string) (hostclaim.Observed, error) { |
| + r, err := hostops.ReadRecord(ctx, c.Store, key) |
| + if err != nil { |
| + return hostclaim.Observed{}, err |
| + } |
| + obs, err := hostclaim.Read(ctx, c.Store, r.Holder.Host) |
| + if err != nil { |
| + return obs, err |
| + } |
| + if obs.Absent { |
| + return obs, hostops.ErrPending |
| + } |
| + if obs.Claim.Holder == nil || obs.Claim.Holder.Key != key { |
| + return obs, hostops.ErrIdentity |
| + } |
| + return obs, nil |
| +} |
diff --git a/internal/console/hostclaimops.go b/internal/console/hostclaimops.go| index 1f70ef3f1..ef9aa7647 100644 |
| --- a/internal/console/hostclaimops.go |
| +++ b/internal/console/hostclaimops.go |
| @@ -35,14 +35,14 @@ func (s *Server) StartClaimRecoveryPass(ctx context.Context) { |
| if !s.cfg.HostClaim { |
| return |
| } |
| - if s.cfg.HostClaimRecovery == nil { |
| + if s.cfg.HostClaimOps == nil { |
| log.Printf("console: claim recovery: %v", hostops.ErrContract) |
| return |
| } |
| t := time.NewTicker(claimRecoveryInterval) |
| defer t.Stop() |
| for { |
| - if err := s.claimRecoveryPass(ctx); err != nil { |
| + if err := s.RunClaimRecoveryPass(ctx); err != nil { |
| log.Printf("console: claim recovery: %v", err) |
| } |
| select { |
diff --git a/internal/domains/claims.go b/internal/domains/claims.go| index a31cfadb2..000528466 100644 |
| --- a/internal/domains/claims.go |
| +++ b/internal/domains/claims.go |
| @@ -35,13 +35,16 @@ func ClaimKey(host string) string { return ClaimsPrefix + host + ".json" } |
| // CreatedBy is omitempty because a claim minted by a box-side process, not a |
| // human door, has no operator to name. |
| type Claim struct { |
| - Schema int `json:"schema"` // 1 |
| - Host string `json:"host"` |
| - Account string `json:"account"` |
| - App string `json:"app"` |
| - Route string `json:"route,omitempty"` |
| - CreatedAt string `json:"created_at"` |
| - CreatedBy string `json:"created_by,omitempty"` |
| + Acquisition string `json:"acq,omitempty"` |
| + RecordIncarnation string `json:"incarnation,omitempty"` |
| + OwnershipState string `json:"ownership_state,omitempty"` |
| + Schema int `json:"schema"` // 1 |
| + Host string `json:"host"` |
| + Account string `json:"account"` |
| + App string `json:"app"` |
| + Route string `json:"route,omitempty"` |
| + CreatedAt string `json:"created_at"` |
| + CreatedBy string `json:"created_by,omitempty"` |
| // Wildcard and Catchall are the woodstar-carloku branch's two claim |
| // capabilities (its migration spec §4-5 gives them meaning; nothing in |
| // this package acts on either). They are deliberately orthogonal: |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 3e1c577e8..2c6f45185 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -547,7 +547,7 @@ func runAgent(args []string, buildClaims ClaimFactory) { |
| // by CARLOS_HOST_METRICS — one log line and nothing else where unset. |
| go memwatchLoop() |
| go domainsLoop(reg, peekAcmeCache(args)) |
| - go instancesLoop(reg) |
| + go instancesLoop(reg, claims.Writes) |
| // The deployment-global election, armed BEFORE the loops it gates so a |
| // single-box deployment is holding well before their first tick (a |
| // minute away) — see housekeeper.go. Not a `go` of its own: it publishes |
| @@ -2116,7 +2116,11 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu |
| } |
| defer func(start time.Time) { logSlow("serial pass stages", start, strings.Join(took, ", ")) }(time.Now()) |
| - if err := reconcileSerialized(ctx, store, reg, provisionTickConfig(os.Getenv)); err != nil { |
| + pcfg := provisionTickConfig(os.Getenv) |
| + if err := reconcileBackfill(ctx, writes, store, pcfg.ReleasePub, os.Getenv("CARLOS_DEPLOYMENT_BUCKET")); err != nil { |
| + log.Printf("agent: claim backfill: %v", err) |
| + } |
| + if err := reconcileSerialized(ctx, store, reg, pcfg); err != nil { |
| log.Printf("agent: fast converge reconcile: %v", err) |
| } |
| mark("reconcile") |
| @@ -2671,7 +2675,7 @@ func domainsDuty(pass func(context.Context) error) housekeeperDuty { |
| // already had a store+reg in scope) — that coupled binding materialization |
| // to PLATFORM_APPS_DOMAIN being set, which has nothing to do with fleets; |
| // this loop has no such gate and shares the same store already opened above. |
| -func instancesLoop(reg *registry.Registry) { |
| +func instancesLoop(reg *registry.Registry, writes hostwrite.Config) { |
| ctx := context.Background() |
| store, err := openDeployStore(ctx) |
| if err != nil { |
| @@ -2692,6 +2696,9 @@ func instancesLoop(reg *registry.Registry) { |
| for { |
| time.Sleep(delay) |
| delay = 5*time.Minute + rand.N(time.Minute) |
| + if err := reconcileBackfill(ctx, writes, store, cfg.ReleasePub, os.Getenv("CARLOS_DEPLOYMENT_BUCKET")); err != nil { |
| + log.Printf("agent: claim backfill: %v", err) |
| + } |
| if err := reconcileSerialized(ctx, store, reg, cfg); err != nil { |
| log.Printf("agent: instances pass: %v", err) |
| } |
diff --git a/internal/host/claimbackfill.go b/internal/host/claimbackfill.go| new file mode 100644 |
| index 000000000..d751062f8 |
| --- /dev/null |
| +++ b/internal/host/claimbackfill.go |
| @@ -0,0 +1,32 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "crypto/ed25519" |
| + "crypto/rand" |
| + "encoding/hex" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostops" |
| + "github.com/carlosframework/platform/internal/hostwrite" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +func reconcileBackfill(ctx context.Context, writes hostwrite.Config, s release.Store, pub ed25519.PublicKey, bucket string) error { |
| + if !writes.Enabled { |
| + return nil |
| + } |
| + if s == nil || len(pub) == 0 || bucket == "" { |
| + return hostops.ErrContract |
| + } |
| + d := hostactors.Directory{Store: s, PublicKey: pub, Bucket: bucket} |
| + c := hostops.NewBackfillController(s, d, time.Now, func() string { |
| + var b [16]byte |
| + if _, err := rand.Read(b[:]); err != nil { |
| + return "" |
| + } |
| + return hex.EncodeToString(b[:]) |
| + }) |
| + return c.ReconcileClaims(ctx) |
| +} |
diff --git a/internal/hostactors/censuskey.go b/internal/hostactors/censuskey.go| new file mode 100644 |
| index 000000000..f0056fa4c |
| --- /dev/null |
| +++ b/internal/hostactors/censuskey.go |
| @@ -0,0 +1,59 @@ |
| +package hostactors |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "crypto/ed25519" |
| + "encoding/json" |
| + "math" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// CensusKeyBinding is approved by the root enrollment coordinator after its |
| +// authenticated box session supplies a proof of possession. It is not an |
| +// unauthenticated enrollment request or a console-generated box report. |
| +type CensusKeyBinding struct { |
| + Bucket string `json:"bucket"` |
| + Box string `json:"box"` |
| + Enrollment string `json:"enrollment"` |
| + PublicKey ed25519.PublicKey `json:"public_key"` |
| +} |
| + |
| +func (d Directory) BindCensusKey(ctx context.Context, signer release.Signer, before Observed, binding CensusKeyBinding, proof []byte) (Observed, error) { |
| + b, ok := before.Roster.Boxes[binding.Box] |
| + if !ok || binding.Bucket != d.Bucket || binding.Enrollment != b.Enrollment || len(binding.PublicKey) != ed25519.PublicKeySize || before.ETag == "" || before.Roster.Revision == math.MaxUint64 { |
| + return Observed{}, ErrRoster |
| + } |
| + raw, err := json.Marshal(binding) |
| + if err != nil { |
| + return Observed{}, err |
| + } |
| + if !ed25519.Verify(binding.PublicKey, raw, proof) { |
| + return Observed{}, ErrRoster |
| + } |
| + if len(b.CensusPublicKey) != 0 { |
| + if bytes.Equal(b.CensusPublicKey, binding.PublicKey) { |
| + return before, nil |
| + } |
| + return Observed{}, ErrRoster |
| + } |
| + // One write-once extension to an existing enrollment. Ordinary Replace |
| + // continues to reject changes to this field, including deletion/rotation. |
| + next := Clone(before.Roster) |
| + b.CensusPublicKey = append(ed25519.PublicKey(nil), binding.PublicKey...) |
| + next.Boxes[binding.Box] = b |
| + next.Revision++ |
| + if err = next.Validate(d.Bucket); err != nil { |
| + return Observed{}, err |
| + } |
| + wire, err := release.SignRootEnvelope(signer, next) |
| + if err != nil { |
| + return Observed{}, err |
| + } |
| + tag, err := d.Store.Put(ctx, Key, wire, before.ETag) |
| + if err != nil { |
| + return Observed{}, err |
| + } |
| + return Observed{Roster: next, ETag: tag}, nil |
| +} |
diff --git a/internal/hostactors/roster.go b/internal/hostactors/roster.go| index 7ef67ddb5..42bd34b19 100644 |
| --- a/internal/hostactors/roster.go |
| +++ b/internal/hostactors/roster.go |
| @@ -23,17 +23,18 @@ var token = regexp.MustCompile(`^[0-9a-f]{32}$`) |
| type Cloud = boxid.Cloud |
| type Box struct { |
| - Enrollment string `json:"enrollment"` |
| - Namespace string `json:"namespace"` |
| - BoxID string `json:"box_id"` |
| - Label string `json:"label"` |
| - Region string `json:"region"` |
| - Pool string `json:"pool"` |
| - Cloud Cloud `json:"cloud"` |
| - EvidenceSHA256 string `json:"evidence_sha256"` |
| - State string `json:"state"` |
| - QuarantineRevision uint64 `json:"quarantine_revision"` |
| - LossIDs []string `json:"loss_ids"` |
| + CensusPublicKey ed25519.PublicKey `json:"census_public_key,omitempty"` |
| + Enrollment string `json:"enrollment"` |
| + Namespace string `json:"namespace"` |
| + BoxID string `json:"box_id"` |
| + Label string `json:"label"` |
| + Region string `json:"region"` |
| + Pool string `json:"pool"` |
| + Cloud Cloud `json:"cloud"` |
| + EvidenceSHA256 string `json:"evidence_sha256"` |
| + State string `json:"state"` |
| + QuarantineRevision uint64 `json:"quarantine_revision"` |
| + LossIDs []string `json:"loss_ids"` |
| } |
| type Console struct { |
| Compute *ComputeBinding `json:"compute,omitempty"` |
| @@ -94,6 +95,9 @@ func (r Roster) Validate(bucket string) error { |
| ids := map[string]bool{} |
| clouds := map[Cloud]bool{} |
| for key, b := range r.Boxes { |
| + if len(b.CensusPublicKey) != 0 && len(b.CensusPublicKey) != ed25519.PublicKeySize { |
| + return ErrRoster |
| + } |
| if !segment(key) || b.BoxID != key || !token.MatchString(b.Enrollment) || ids[b.Enrollment] || b.Namespace == "" || b.Label == "" || b.Region == "" || b.Pool == "" || b.Cloud.Provider != "aws" || b.Cloud.Account == "" || b.Cloud.Region == "" || b.Cloud.InstanceID == "" || len(b.EvidenceSHA256) != 64 || clouds[b.Cloud] { |
| return ErrRoster |
| } |
diff --git a/internal/hostclaim/adopt.go b/internal/hostclaim/adopt.go| index d4492f7a6..bc7ff4cf5 100644 |
| --- a/internal/hostclaim/adopt.go |
| +++ b/internal/hostclaim/adopt.go |
| @@ -16,7 +16,7 @@ func Adopt(nh NewHolder, servingBox string, otherRegions, otherBoxes []string, o |
| if err != nil { |
| return err |
| } |
| - c.Holder = &Holder{Family: nh.Family, Key: nh.Key, Account: nh.Account, App: nh.App, Name: nh.Name, |
| + c.Holder = &Holder{Acquisition: nh.Acquisition, Family: nh.Family, Key: nh.Key, Account: nh.Account, App: nh.App, Name: nh.Name, |
| Incarnation: inc, CreatedAt: nh.CreatedAt, Location: "instances/" + nh.Host, Stamp: c.Gen + 1, WriterBox: servingBox} |
| c.Op = Op{Kind: KindServing, Exec: servingBox, Participants: []string{servingBox}} |
| c.Touched = Touched{Complete: true, Regions: []string{nh.Region}, Boxes: []string{servingBox}} |
diff --git a/internal/hostclaim/claim.go b/internal/hostclaim/claim.go| index 698444fe1..23d178589 100644 |
| --- a/internal/hostclaim/claim.go |
| +++ b/internal/hostclaim/claim.go |
| @@ -37,7 +37,7 @@ const ( |
| var knownKinds = map[string]bool{KindServing: true, KindCreating: true, KindDeleting: true, KindParking: true, |
| KindPurging: true, KindPlacing: true, KindMoving: true, KindLanding: true, KindReverting: true, |
| - KindRestoring: true, KindPushing: true} |
| + KindRestoring: true, KindPushing: true, KindRelocating: true} |
| var ( |
| ErrMalformed = errors.New("hostclaim: malformed claim") |
| @@ -45,6 +45,7 @@ var ( |
| ) |
| type Holder struct { |
| + Acquisition string `json:"acq,omitempty"` |
| Family string `json:"family"` // "instance" | "sidecar" |
| Key string `json:"key"` // exact record key |
| Account string `json:"account"` |
| @@ -221,6 +222,9 @@ func (c *Claim) Validate() error { |
| if c == nil || c.V != schemaVersion || !controlname.ValidHostShape(c.Host) || c.Gen == 0 { |
| return fmt.Errorf("%w: header", ErrMalformed) |
| } |
| + if err := validateRelocating(c); err != nil { |
| + return err |
| + } |
| if c.Op.Kind != "" && !knownKinds[c.Op.Kind] { |
| return fmt.Errorf("%w: unknown op kind %q", ErrMalformed, c.Op.Kind) |
| } |
diff --git a/internal/hostclaim/create.go b/internal/hostclaim/create.go| index 4450413ba..c8fe97cc2 100644 |
| --- a/internal/hostclaim/create.go |
| +++ b/internal/hostclaim/create.go |
| @@ -10,6 +10,7 @@ import ( |
| var ErrRetiredIncarnation = errors.New("hostclaim: incarnation was retired on this host") |
| type NewHolder struct { |
| + Acquisition string |
| Family, Key, Account, App, Name, Host string |
| RecordRegion string // Region encoded in the record; empty means home region. |
| Region string // Resolved storage region; must be non-empty. |
| @@ -70,7 +71,7 @@ func BeginCreate(nh NewHolder, recordBody []byte, opID, by string, now time.Time |
| return ErrRetiredIncarnation |
| } |
| } |
| - c.Holder = &Holder{Family: nh.Family, Key: nh.Key, Account: nh.Account, App: nh.App, Name: nh.Name, |
| + c.Holder = &Holder{Acquisition: nh.Acquisition, Family: nh.Family, Key: nh.Key, Account: nh.Account, App: nh.App, Name: nh.Name, |
| Incarnation: inc, CreatedAt: nh.CreatedAt, Location: "instances/" + nh.Host + "/" + inc} |
| c.Op = Op{Kind: KindCreating, ID: opID, By: by, StartedAt: now.UTC(), |
| Intent: map[string]string{"key": nh.Key, "record": string(recordBody)}} |
diff --git a/internal/hostclaim/fence.go b/internal/hostclaim/fence.go| new file mode 100644 |
| index 000000000..bb90dd9bf |
| --- /dev/null |
| +++ b/internal/hostclaim/fence.go |
| @@ -0,0 +1,109 @@ |
| +package hostclaim |
| + |
| +import ( |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "errors" |
| + "strings" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/termfence" |
| +) |
| + |
| +var ErrFence = errors.New("hostclaim: invalid termination fence") |
| + |
| +type FenceRef struct { |
| + Enrollment string `json:"enrollment"` |
| + Key string `json:"key"` |
| + SHA256 string `json:"sha256"` |
| +} |
| + |
| +type FenceBinding struct { |
| + Region string |
| + RosterRevision uint64 |
| + RosterETag string |
| +} |
| + |
| +// FenceAuthority checks and identifies one authenticated roster observation. |
| +// Implementations must bind both the target and reporting worker enrollments, |
| +// cloud account, cloud region and terminal instance identity to that roster. |
| +type FenceAuthority interface { |
| + ValidateFence(context.Context, termfence.Record) (FenceBinding, error) |
| +} |
| + |
| +type VerifiedFence struct { |
| + ref FenceRef |
| + record termfence.Record |
| + binding FenceBinding |
| +} |
| + |
| +func validFenceRef(ref FenceRef) bool { |
| + if len(ref.Enrollment) != 32 || ref.Enrollment != strings.ToLower(ref.Enrollment) { |
| + return false |
| + } |
| + if _, err := hex.DecodeString(ref.Enrollment); err != nil { |
| + return false |
| + } |
| + return ref.Key == "control/box-fences/"+ref.Enrollment+".json" && isHex64(ref.SHA256) && ref.SHA256 == strings.ToLower(ref.SHA256) |
| +} |
| + |
| +// VerifyFence re-reads the immutable worker-owned object, including on replay. |
| +// A previously verified reference alone never substitutes for verification. |
| +func VerifyFence(ctx context.Context, s release.Store, authority FenceAuthority, ref FenceRef) (VerifiedFence, error) { |
| + if s == nil || authority == nil || !validFenceRef(ref) { |
| + return VerifiedFence{}, ErrFence |
| + } |
| + raw, etag, err := s.Get(ctx, ref.Key) |
| + if err != nil { |
| + return VerifiedFence{}, err |
| + } |
| + sum := sha256.Sum256(raw) |
| + if etag == "" || hex.EncodeToString(sum[:]) != ref.SHA256 { |
| + return VerifiedFence{}, ErrFence |
| + } |
| + var f termfence.Record |
| + if json.Unmarshal(raw, &f) != nil || f.Enrollment != ref.Enrollment || f.Box == "" || f.State != "terminated" || f.V != 1 || f.WrittenBy.Box == f.Box { |
| + return VerifiedFence{}, ErrFence |
| + } |
| + binding, err := authority.ValidateFence(ctx, f) |
| + if err != nil { |
| + return VerifiedFence{}, err |
| + } |
| + if binding.Region == "" || binding.RosterRevision == 0 || binding.RosterETag == "" { |
| + return VerifiedFence{}, ErrFence |
| + } |
| + return VerifiedFence{ref: ref, record: f, binding: binding}, nil |
| +} |
| + |
| +// VerifyOperationFence binds the journal key as well as the immutable record. |
| +// Return and recovery paths use this form rather than trusting a decoded ref. |
| +func VerifyOperationFence(ctx context.Context, s release.Store, authority FenceAuthority, op Op, box string) error { |
| + if !validBoxReference(box) || op.ID == "" { |
| + return ErrFence |
| + } |
| + var ref FenceRef |
| + if json.Unmarshal([]byte(op.Journal["fence."+box]), &ref) != nil { |
| + return ErrFence |
| + } |
| + verified, err := VerifyFence(ctx, s, authority, ref) |
| + if err != nil { |
| + return err |
| + } |
| + if verified.record.Box != box { |
| + return ErrFence |
| + } |
| + return nil |
| +} |
| + |
| +func VerifyFenceForBox(ctx context.Context, s release.Store, authority FenceAuthority, ref FenceRef, box string) error { |
| + verified, err := VerifyFence(ctx, s, authority, ref) |
| + if err != nil { |
| + return err |
| + } |
| + if !validBoxReference(box) || verified.record.Box != box { |
| + return ErrFence |
| + } |
| + return nil |
| +} |
diff --git a/internal/hostclaim/relocating.go b/internal/hostclaim/relocating.go| new file mode 100644 |
| index 000000000..b5565460c |
| --- /dev/null |
| +++ b/internal/hostclaim/relocating.go |
| @@ -0,0 +1,142 @@ |
| +package hostclaim |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "fmt" |
| + "strconv" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +const KindRelocating = "relocating" |
| + |
| +// PreFlipGuard verifies the durable pre-flip conversion decision in the |
| +// authoritative instance record. That decision's record CAS competes with |
| +// MoveFlip and makes the move terminal for every cutting/flip transition. |
| +// It preserves the old move journal for replay, and binds this op, incarnation, |
| +// source and exact fence reference. A snapshot of an undecided record is not |
| +// an implementation of this interface. |
| +type PreFlipGuard interface { |
| + CheckMovingBeforeFlip(context.Context, Observed, string, FenceRef) (string, error) |
| +} |
| + |
| +// RelocationRecoveryOwner reports replay ownership, never execution authority. |
| +func RelocationRecoveryOwner(c *Claim) (string, error) { |
| + if c == nil || c.Op.Kind != KindRelocating || c.Op.Intent["region"] == "" { |
| + return "", ErrWrongState |
| + } |
| + return "region:" + c.Op.Intent["region"], nil |
| +} |
| + |
| +func validateRelocating(c *Claim) error { |
| + if c.Op.Kind != KindRelocating { |
| + return nil |
| + } |
| + if c.Holder == nil || c.Op.Intent["region"] == "" || !validBoxReference(c.Op.Intent["source_box"]) || c.Op.Intent["source_box"] == "none" { |
| + return ErrMalformed |
| + } |
| + switch c.Op.Phase { |
| + case "starting", "releasing": |
| + if c.Op.Exec != c.Op.Intent["source_box"] && c.Op.Exec != "none" { |
| + return ErrMalformed |
| + } |
| + case "pinning": |
| + if c.Op.Exec != "none" { |
| + return ErrMalformed |
| + } |
| + case "installing", "installed", "probing", "probe-failed", "serving": |
| + if !validBoxReference(c.Op.Exec) || c.Op.Exec == "none" || c.Op.Exec != c.Op.Intent["target_box"] { |
| + return ErrMalformed |
| + } |
| + default: |
| + return ErrMalformed |
| + } |
| + return nil |
| +} |
| + |
| +// ConvertMovingToRelocating performs exactly one claim CAS. It deliberately |
| +// exposes no transition capable of granting relocation execution. Callers must |
| +// be claim-enabled and first persist the record conversion decision. |
| +func ConvertMovingToRelocating(ctx context.Context, s release.Store, observed Observed, opID string, ref FenceRef, authority FenceAuthority, guard PreFlipGuard) (Observed, error) { |
| + if guard == nil || observed.Absent || observed.Claim == nil || observed.ETag == "" { |
| + return Observed{}, ErrWrongState |
| + } |
| + before := observed.Claim |
| + if opID == "" || before.Op.ID != opID { |
| + return Observed{}, ErrStaleOp |
| + } |
| + if before.Holder == nil || before.Op.Intent["admission_id"] == "" || before.Holder.Family != "instance" || before.Op.Kind != KindMoving || before.Op.Phase == "landing" || before.Op.Phase == "committed" { |
| + return Observed{}, ErrWrongState |
| + } |
| + // Check the instance record even while the claim still says moving: the |
| + // instance flip is earlier than final commit and can precede claim landing. |
| + region, err := guard.CheckMovingBeforeFlip(ctx, observed, opID, ref) |
| + if err != nil { |
| + return Observed{}, err |
| + } |
| + fence, err := VerifyFence(ctx, s, authority, ref) |
| + if err != nil { |
| + return Observed{}, err |
| + } |
| + if region == "" || region != fence.binding.Region || before.Op.Intent["source_box"] != fence.record.Box || before.Op.Intent["source_enrollment"] != ref.Enrollment || before.Op.Intent["source_region"] != region || (before.Op.Exec != fence.record.Box && before.Op.Exec != "none" && before.Op.Exec != "") { |
| + return Observed{}, ErrFence |
| + } |
| + raw, err := json.Marshal(ref) |
| + if err != nil { |
| + return Observed{}, err |
| + } |
| + return Update(ctx, s, observed, func(c *Claim) error { |
| + if c.Op.ID != opID || c.Op.Kind != KindMoving || c.Op.Phase != before.Op.Phase { |
| + return ErrStaleOp |
| + } |
| + key := "fence." + fence.record.Box |
| + if old, ok := c.Op.Journal[key]; ok && old != string(raw) { |
| + return fmt.Errorf("%w: fence reference already bound", ErrFence) |
| + } |
| + // Recheck the durable record decision immediately before the claim CAS. |
| + // Its terminal state prevents any intervening flip. |
| + currentRegion, err := guard.CheckMovingBeforeFlip(ctx, observed, opID, ref) |
| + if err != nil { |
| + return err |
| + } |
| + if currentRegion != region { |
| + return ErrWrongState |
| + } |
| + if c.Op.Journal == nil { |
| + c.Op.Journal = map[string]string{} |
| + } |
| + c.Op.Journal[key] = string(raw) |
| + c.Op.Journal["reason"] = "source-terminated-before-flip" |
| + c.Op.Journal["source_stamp"] = strconv.FormatUint(c.Holder.Stamp, 10) |
| + c.Op.Journal["source_generation"] = c.Holder.Generation |
| + c.Op.Journal["roster_revision"] = strconv.FormatUint(fence.binding.RosterRevision, 10) |
| + c.Op.Journal["roster_etag"] = fence.binding.RosterETag |
| + c.Op.Journal["admission_id"] = before.Op.Intent["admission_id"] |
| + c.Op.Journal["conversion_claim_gen"] = strconv.FormatUint(before.Gen, 10) |
| + c.Op.Journal["conversion_claim_etag"] = observed.ETag |
| + c.Op.Kind, c.Op.Phase, c.Op.Exec = KindRelocating, "pinning", "none" |
| + if c.Op.Intent == nil { |
| + c.Op.Intent = map[string]string{} |
| + } |
| + c.Op.Intent["region"] = region |
| + // The original move target is in a different region. It is not a selected |
| + // same-region replacement. Keep the move history without authorizing it. |
| + if target := c.Op.Intent["target_box"]; target != "" { |
| + c.Op.Intent["move_target_box"] = target |
| + delete(c.Op.Intent, "target_box") |
| + } |
| + if enrollment := c.Op.Intent["target_enrollment"]; enrollment != "" { |
| + c.Op.Intent["move_target_enrollment"] = enrollment |
| + delete(c.Op.Intent, "target_enrollment") |
| + } |
| + participants := make([]string, 0, len(c.Op.Participants)) |
| + for _, box := range c.Op.Participants { |
| + if box != fence.record.Box { |
| + participants = append(participants, box) |
| + } |
| + } |
| + c.Op.Participants = participants |
| + return nil |
| + }) |
| +} |
diff --git a/internal/hostclaim/relocating_test.go b/internal/hostclaim/relocating_test.go| new file mode 100644 |
| index 000000000..88d2cf2a7 |
| --- /dev/null |
| +++ b/internal/hostclaim/relocating_test.go |
| @@ -0,0 +1,251 @@ |
| +package hostclaim |
| + |
| +import ( |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "errors" |
| + "reflect" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/boxid" |
| + "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/termfence" |
| +) |
| + |
| +type c7Authority struct { |
| + want termfence.Record |
| + err error |
| +} |
| + |
| +func (a c7Authority) ValidateFence(_ context.Context, f termfence.Record) (FenceBinding, error) { |
| + if a.err != nil { |
| + return FenceBinding{}, a.err |
| + } |
| + if f != a.want { |
| + return FenceBinding{}, ErrFence |
| + } |
| + return FenceBinding{Region: "ie", RosterRevision: 7, RosterETag: "roster-etag"}, nil |
| +} |
| + |
| +type c7Guard struct { |
| + calls int |
| + rejectAt int |
| +} |
| + |
| +func (g *c7Guard) CheckMovingBeforeFlip(_ context.Context, o Observed, id string, ref FenceRef) (string, error) { |
| + g.calls++ |
| + if id != o.Claim.Op.ID || ref.Enrollment != o.Claim.Op.Intent["source_enrollment"] { |
| + return "", ErrFence |
| + } |
| + if g.rejectAt > 0 && g.calls >= g.rejectAt { |
| + return "", ErrWrongState |
| + } |
| + return "ie", nil |
| +} |
| + |
| +type c7Store struct { |
| + release.Store |
| + puts int |
| +} |
| + |
| +func (s *c7Store) Put(ctx context.Context, key string, body []byte, match string) (string, error) { |
| + s.puts++ |
| + return s.Store.Put(ctx, key, body, match) |
| +} |
| +func c7Fixture(t *testing.T) (*c7Store, Observed, FenceRef, c7Authority) { |
| + t.Helper() |
| + ctx := context.Background() |
| + s := &c7Store{Store: release.NewDirStore(t.TempDir())} |
| + o := servingClaim(t, ctx, s) |
| + var err error |
| + o, err = Update(ctx, s, o, func(c *Claim) error { |
| + c.Holder.Generation, c.Holder.Stamp = "gen-current", 17 |
| + c.Op = Op{ID: "move-1", Kind: KindMoving, Phase: "releasing", Exec: "adi-knitting", |
| + Intent: map[string]string{"admission_id": "admission-existing", "source_box": "adi-knitting", "source_enrollment": strings.Repeat("a", 32), "source_region": "ie", "target_box": "bri-knitting", "target_enrollment": strings.Repeat("c", 32)}, |
| + Journal: map[string]string{"pin": "keep"}, Participants: []string{"adi-knitting", "bri-knitting"}} |
| + c.Obligations = []Obligation{{ID: "staging", Op: "move-1", Kind: "park-replica", Region: "uk", Owner: "region:uk", Retirement: "retire-1", Park: &ParkScope{Location: "moves/" + testHost + "/move-1", Incarnation: c.Holder.Incarnation, EvidenceAt: time.Now().UTC(), RequestLimit: 120 * time.Second}}} |
| + return nil |
| + }) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + f := termfence.Record{V: 1, Box: "adi-knitting", Enrollment: strings.Repeat("a", 32), Cloud: boxid.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-source"}, State: "terminated", ObservedAt: "2026-10-06T00:00:00Z", Via: "describe", RequestID: "request", WrittenBy: termfence.WrittenBy{Box: "worker", Enrollment: strings.Repeat("b", 32)}} |
| + raw, err := json.Marshal(f) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + digest := sha256.Sum256(raw) |
| + ref := FenceRef{Enrollment: f.Enrollment, Key: "control/box-fences/" + f.Enrollment + ".json", SHA256: hex.EncodeToString(digest[:])} |
| + if _, err := s.Put(ctx, ref.Key, raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + s.puts = 0 |
| + return s, o, ref, c7Authority{want: f} |
| +} |
| + |
| +func TestC7ConversionPreservesOwnershipAndChores(t *testing.T) { |
| + s, before, ref, a := c7Fixture(t) |
| + guard := &c7Guard{} |
| + after, err := ConvertMovingToRelocating(context.Background(), s, before, "move-1", ref, a, guard) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + c := after.Claim |
| + if c.Op.Journal["admission_id"] != "admission-existing" || c.Op.Journal["source_stamp"] != "17" || c.Op.Journal["source_generation"] != "gen-current" || c.Op.Journal["roster_revision"] != "7" || c.Op.Journal["roster_etag"] != "roster-etag" || c.Op.Journal["reason"] != "source-terminated-before-flip" || c.Op.Journal["conversion_claim_etag"] != before.ETag { |
| + t.Fatalf("missing conversion authority: %+v", c.Op.Journal) |
| + } |
| + if s.puts != 1 || c.Gen != before.Claim.Gen+1 || guard.calls != 2 { |
| + t.Fatalf("writes=%d gen=%d checks=%d", s.puts, c.Gen, guard.calls) |
| + } |
| + if !reflect.DeepEqual(c.Holder, before.Claim.Holder) || !reflect.DeepEqual(c.Obligations, before.Claim.Obligations) { |
| + t.Fatal("holder or staging retirement changed") |
| + } |
| + if c.Op.ID != "move-1" || c.Op.Kind != KindRelocating || c.Op.Phase != "pinning" || c.Op.Exec != "none" || c.Op.Intent["region"] != "ie" { |
| + t.Fatalf("wrong conversion: %+v", c.Op) |
| + } |
| + if c.Op.Intent["target_box"] != "" || c.Op.Intent["move_target_box"] != "bri-knitting" || !reflect.DeepEqual(c.Op.Participants, []string{"bri-knitting"}) { |
| + t.Fatalf("target or barrier: %+v", c.Op) |
| + } |
| + var saved FenceRef |
| + if err := json.Unmarshal([]byte(c.Op.Journal["fence.adi-knitting"]), &saved); err != nil || saved != ref { |
| + t.Fatalf("journal reference: %+v %v", saved, err) |
| + } |
| + if owner, err := RelocationRecoveryOwner(c); err != nil || owner != "region:ie" { |
| + t.Fatalf("owner=%q error=%v", owner, err) |
| + } |
| + if _, err := ConvertMovingToRelocating(context.Background(), s, before, "move-1", ref, a, &c7Guard{}); !errors.Is(err, ErrConflict) { |
| + t.Fatalf("stale replay: %v", err) |
| + } |
| +} |
| + |
| +func TestC7ConversionRefusesWithoutWriting(t *testing.T) { |
| + for _, tc := range []string{"landing-record", "flip-during-verify", "landing-claim", "wrong-op", "wrong-source", "wrong-enrollment", "wrong-region", "foreign-exec", "journal-rebind", "authority", "digest", "no-guard", "missing-admission"} { |
| + t.Run(tc, func(t *testing.T) { |
| + s, o, ref, a := c7Fixture(t) |
| + id := "move-1" |
| + var guard PreFlipGuard = &c7Guard{} |
| + switch tc { |
| + case "landing-record": |
| + guard = &c7Guard{rejectAt: 1} |
| + case "flip-during-verify": |
| + guard = &c7Guard{rejectAt: 2} |
| + case "landing-claim": |
| + o.Claim.Op.Kind = KindLanding |
| + case "wrong-op": |
| + id = "other" |
| + case "wrong-source": |
| + o.Claim.Op.Intent["source_box"] = "other" |
| + case "wrong-enrollment": |
| + o.Claim.Op.Intent["source_enrollment"] = strings.Repeat("d", 32) |
| + case "wrong-region": |
| + o.Claim.Op.Intent["source_region"] = "uk" |
| + case "foreign-exec": |
| + o.Claim.Op.Exec = "target" |
| + case "journal-rebind": |
| + o.Claim.Op.Journal["fence.adi-knitting"] = `{"enrollment":"other"}` |
| + case "authority": |
| + a.err = ErrFence |
| + case "digest": |
| + ref.SHA256 = strings.Repeat("0", 64) |
| + case "no-guard": |
| + guard = nil |
| + case "missing-admission": |
| + delete(o.Claim.Op.Intent, "admission_id") |
| + } |
| + raw, _ := json.Marshal(o.Claim) |
| + if _, err := ConvertMovingToRelocating(context.Background(), s, o, id, ref, a, guard); err == nil { |
| + t.Fatal("accepted") |
| + } |
| + unchanged, _ := json.Marshal(o.Claim) |
| + if s.puts != 0 || string(raw) != string(unchanged) { |
| + t.Fatal("refusal mutated claim") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestC7VerifyFenceReReadsExactObject(t *testing.T) { |
| + ctx := context.Background() |
| + s, _, ref, a := c7Fixture(t) |
| + if _, err := VerifyFence(ctx, s, a, ref); err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, tc := range []string{"bad-key", "bad-enrollment", "bad-digest", "missing", "changed-bytes", "authority"} { |
| + t.Run(tc, func(t *testing.T) { |
| + s, _, ref, a := c7Fixture(t) |
| + switch tc { |
| + case "bad-key": |
| + ref.Key = "control/box-fences/other.json" |
| + case "bad-enrollment": |
| + ref.Enrollment = "../escape" |
| + case "bad-digest": |
| + ref.SHA256 = strings.Repeat("f", 64) |
| + case "missing": |
| + if err := s.Delete(ctx, ref.Key); err != nil { |
| + t.Fatal(err) |
| + } |
| + case "changed-bytes": |
| + raw, etag, err := s.Get(ctx, ref.Key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err = s.Put(ctx, ref.Key, append(raw, ' '), etag); err != nil { |
| + t.Fatal(err) |
| + } |
| + case "authority": |
| + a.err = ErrFence |
| + } |
| + if _, err := VerifyFence(ctx, s, a, ref); err == nil { |
| + t.Fatal("accepted") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestC7RelocatingPhaseValidation(t *testing.T) { |
| + for _, phase := range []string{"starting", "releasing", "pinning", "installing", "installed", "probing", "probe-failed", "serving", "unknown"} { |
| + t.Run(phase, func(t *testing.T) { |
| + _, o, _, _ := c7Fixture(t) |
| + c := o.Claim |
| + c.Op.Kind, c.Op.Phase = KindRelocating, phase |
| + c.Op.Intent["region"] = "ie" |
| + switch phase { |
| + case "starting", "releasing": |
| + c.Op.Exec = "adi-knitting" |
| + case "pinning": |
| + c.Op.Exec = "none" |
| + default: |
| + c.Op.Exec = "bri-knitting" |
| + } |
| + if err := c.Validate(); (err != nil) != (phase == "unknown") { |
| + t.Fatalf("phase %s: %v", phase, err) |
| + } |
| + if phase == "pinning" { |
| + c.Op.Exec = "bri-knitting" |
| + if c.Validate() == nil { |
| + t.Fatal("pinning granted execution") |
| + } |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestC7JournalFenceCannotNameAnotherBox(t *testing.T) { |
| + s, o, ref, a := c7Fixture(t) |
| + raw, err := json.Marshal(ref) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + o.Claim.Op.Journal["fence.adi-knitting"] = string(raw) |
| + if err := VerifyOperationFence(context.Background(), s, a, o.Claim.Op, "adi-knitting"); err != nil { |
| + t.Fatal(err) |
| + } |
| + o.Claim.Op.Journal["fence.other"] = string(raw) |
| + if err := VerifyOperationFence(context.Background(), s, a, o.Claim.Op, "other"); err == nil { |
| + t.Fatal("journal key rebound to another box") |
| + } |
| +} |
diff --git a/internal/hostfence/claim_fence.go b/internal/hostfence/claim_fence.go| new file mode 100644 |
| index 000000000..e4e09e6f6 |
| --- /dev/null |
| +++ b/internal/hostfence/claim_fence.go |
| @@ -0,0 +1,38 @@ |
| +package hostfence |
| + |
| +import ( |
| + "context" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/termfence" |
| +) |
| + |
| +// ClaimFenceAuthority authenticates the roster without creating the reverse |
| +// hostclaim -> hostactors dependency. Immutable evidence remains valid after |
| +// its reporter retires; enrollment/cloud binding must still be retained. |
| +type ClaimFenceAuthority struct { |
| + Directory hostactors.Directory |
| + Now func() time.Time |
| +} |
| + |
| +var _ hostclaim.FenceAuthority = ClaimFenceAuthority{} |
| + |
| +func (a ClaimFenceAuthority) ValidateFence(ctx context.Context, f termfence.Record) (hostclaim.FenceBinding, error) { |
| + if a.Now == nil || f.WrittenBy.Box == f.Box { |
| + return hostclaim.FenceBinding{}, hostclaim.ErrFence |
| + } |
| + o, err := a.Directory.Read(ctx) |
| + if err != nil { |
| + return hostclaim.FenceBinding{}, err |
| + } |
| + if err := hostactors.ValidateTerminationFence(o, f.Box, f, a.Now()); err != nil { |
| + return hostclaim.FenceBinding{}, err |
| + } |
| + return hostclaim.FenceBinding{Region: o.Roster.Boxes[f.Box].Region, RosterRevision: o.Roster.Revision, RosterETag: o.ETag}, nil |
| +} |
| + |
| +func ClaimFenceRef(ref EvidenceRef) hostclaim.FenceRef { |
| + return hostclaim.FenceRef{Enrollment: ref.Enrollment, Key: ref.Key, SHA256: ref.SHA256} |
| +} |
diff --git a/internal/hostfence/claim_fence_test.go b/internal/hostfence/claim_fence_test.go| new file mode 100644 |
| index 000000000..6ccaa8ae0 |
| --- /dev/null |
| +++ b/internal/hostfence/claim_fence_test.go |
| @@ -0,0 +1,72 @@ |
| +package hostfence |
| + |
| +import ( |
| + "context" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "testing" |
| + |
| + "github.com/aws/aws-sdk-go-v2/service/ec2/types" |
| + "github.com/carlosframework/platform/internal/awsinv" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| +) |
| + |
| +func TestC7FenceReaderAuthenticatesRosterAndCloud(t *testing.T) { |
| + for _, tc := range []string{"valid", "enrollment", "account", "cloud-region", "instance", "reporter", "nonterminal", "signature", "self-reporter"} { |
| + t.Run(tc, func(t *testing.T) { |
| + ctx := context.Background() |
| + w, s := terminationFixture(t) |
| + w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) { |
| + return terminatedEC2{state: types.InstanceStateNameTerminated, owner: "123"}, nil |
| + } |
| + got, err := w.Describe(ctx, "dead") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + ref := ClaimFenceRef(got) |
| + raw, etag, err := s.Get(ctx, ref.Key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var f Termination |
| + if err = json.Unmarshal(raw, &f); err != nil { |
| + t.Fatal(err) |
| + } |
| + switch tc { |
| + case "enrollment": |
| + f.Enrollment = "cccccccccccccccccccccccccccccccc" |
| + case "account": |
| + f.Cloud.Account = "456" |
| + case "cloud-region": |
| + f.Cloud.Region = "eu-west-2" |
| + case "instance": |
| + f.Cloud.InstanceID = "i-other" |
| + case "reporter": |
| + f.WrittenBy.Enrollment = "cccccccccccccccccccccccccccccccc" |
| + case "nonterminal": |
| + f.State = "shutting-down" |
| + case "self-reporter": |
| + f.WrittenBy.Box = f.Box |
| + f.WrittenBy.Enrollment = f.Enrollment |
| + case "signature": |
| + w.Directory.PublicKey = make([]byte, 32) |
| + } |
| + raw, err = json.Marshal(f) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + // Deliberately replace the object in this local adversarial fixture and |
| + // recompute its digest: a digest match alone must not grant authority. |
| + if _, err = s.Put(ctx, ref.Key, raw, etag); err != nil { |
| + t.Fatal(err) |
| + } |
| + sum := sha256.Sum256(raw) |
| + ref.SHA256 = hex.EncodeToString(sum[:]) |
| + _, err = hostclaim.VerifyFence(ctx, s, ClaimFenceAuthority{Directory: w.Directory, Now: w.Now}, ref) |
| + if (err == nil) != (tc == "valid") { |
| + t.Fatalf("%s: %v", tc, err) |
| + } |
| + }) |
| + } |
| +} |
diff --git a/internal/hostops/backfill.go b/internal/hostops/backfill.go| new file mode 100644 |
| index 000000000..b90607198 |
| --- /dev/null |
| +++ b/internal/hostops/backfill.go |
| @@ -0,0 +1,380 @@ |
| +package hostops |
| + |
| +import ( |
| + "context" |
| + "crypto/ed25519" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "sort" |
| + "sync" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/domains" |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +const BackfillKey = "control/hosts-backfill.json" |
| +const HostsQuarantineKey = "control/hosts-quarantine.json" |
| + |
| +type CensusInstallation struct { |
| + Placement string `json:"placement"` |
| + Host string `json:"host"` |
| + Account string `json:"account"` |
| + App string `json:"app"` |
| + Family string `json:"family"` |
| + Key string `json:"key"` |
| + Incarnation string `json:"incarnation"` |
| + Stamp uint64 `json:"stamp"` |
| + Local bool `json:"local"` |
| + Route bool `json:"route"` |
| +} |
| +type CensusReport struct { |
| + Box string `json:"box"` |
| + Enrollment string `json:"enrollment"` |
| + BootID string `json:"boot_id"` |
| + Reservation string `json:"reservation"` |
| + DirectoryRevision uint64 `json:"directory_revision"` |
| + CapturedAt time.Time `json:"captured_at"` |
| + Complete bool `json:"complete"` |
| + Installations []CensusInstallation `json:"installations"` |
| +} |
| +type Census struct { |
| + Records []Record |
| + Reports map[string]CensusReport |
| + Roster hostactors.Observed |
| +} |
| +type BackfillHost struct { |
| + Key string `json:"key"` |
| + ETag string `json:"etag"` |
| + Acquisition string `json:"acq"` |
| + Region string `json:"region"` |
| + Box string `json:"box"` |
| +} |
| +type BackfillPlan struct { |
| + V int `json:"v"` |
| + ID string `json:"id"` |
| + Phase string `json:"phase"` |
| + DirectoryRevision uint64 `json:"directory_revision"` |
| + CreatedAt time.Time `json:"created_at"` |
| + Reports map[string]string `json:"reports"` |
| + Hosts map[string]BackfillHost `json:"hosts"` |
| +} |
| +type HostsQuarantine struct { |
| + V int `json:"v"` |
| + Hosts map[string]string `json:"hosts"` |
| +} |
| +type CensusEnvelope struct { |
| + Report CensusReport `json:"report"` |
| + Signature []byte `json:"signature"` |
| +} |
| + |
| +type Backfill struct { |
| + mu sync.Mutex |
| + cached *Census |
| + cachedTag string |
| + Store release.Store |
| + Directory hostactors.Directory |
| +} |
| +type ProductionAuthority struct { |
| + Store release.Store |
| + Directory hostactors.Directory |
| + Now func() time.Time |
| + ConsoleID, Enrollment, Namespace string |
| + Signer release.Signer |
| + Evidence hostactors.Evidence |
| + Backfill *Backfill |
| +} |
| + |
| +func reportKey(reservation, box string) (string, error) { |
| + if !safeSegment(reservation) || !safeSegment(box) { |
| + return "", ErrIdentity |
| + } |
| + return "control/host-census/" + reservation + "/" + box + ".json", nil |
| +} |
| +func safeSegment(s string) bool { |
| + if s == "" || s == "." || s == ".." { |
| + return false |
| + } |
| + for _, c := range s { |
| + if c == '/' || c == '\\' || c == 0 { |
| + return false |
| + } |
| + } |
| + return true |
| +} |
| +func CheckHostQuarantine(ctx context.Context, s release.Store, host string) error { |
| + raw, _, err := s.Get(ctx, HostsQuarantineKey) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return nil |
| + } |
| + if err != nil { |
| + return err |
| + } |
| + var q HostsQuarantine |
| + if decode(raw, &q) != nil || q.V != 1 || q.Hosts == nil { |
| + return ErrIdentity |
| + } |
| + if _, ok := q.Hosts[host]; ok { |
| + return ErrPending |
| + } |
| + return nil |
| +} |
| +func (b *Backfill) read(ctx context.Context) (BackfillPlan, string, error) { |
| + var p BackfillPlan |
| + raw, tag, err := b.Store.Get(ctx, BackfillKey) |
| + if err != nil { |
| + return p, "", err |
| + } |
| + if tag == "" || release.VerifyRootEnvelope(b.Directory.PublicKey, raw, &p) != nil || p.V != 1 || !safeSegment(p.ID) || p.Phase != "adopting" || p.CreatedAt.IsZero() || p.Hosts == nil || p.Reports == nil { |
| + return p, "", ErrContract |
| + } |
| + return p, tag, nil |
| +} |
| +func (a ProductionAuthority) census(ctx context.Context, account, app, reservation string, expectedRevision uint64, digests map[string]string) (Census, error) { |
| + out := Census{Reports: map[string]CensusReport{}} |
| + roster, err := a.Directory.Read(ctx) |
| + if err != nil { |
| + return out, err |
| + } |
| + if expectedRevision != 0 && roster.Roster.Revision != expectedRevision { |
| + return out, ErrPending |
| + } |
| + out.Roster = roster |
| + apps, err := Apps(ctx, a.Store) |
| + if err != nil { |
| + return out, err |
| + } |
| + allRecords := map[string]Record{} |
| + for _, p := range apps { |
| + records, err := Records(ctx, a.Store, p.Account, p.Name) |
| + if err != nil { |
| + return out, err |
| + } |
| + for _, r := range records { |
| + if _, ok := allRecords[r.Holder.Host]; ok { |
| + return out, ErrIdentity |
| + } |
| + allRecords[r.Holder.Host] = r |
| + if account == "" || (p.Account == account && p.Name == app) { |
| + out.Records = append(out.Records, r) |
| + } |
| + } |
| + } |
| + for ref, box := range roster.Roster.Boxes { |
| + // Retired membership is retained until its inventory is proven empty. |
| + key, err := reportKey(reservation, ref) |
| + if err != nil { |
| + return out, err |
| + } |
| + raw, tag, err := a.Store.Get(ctx, key) |
| + if err != nil { |
| + return out, err |
| + } |
| + if digests != nil { |
| + sum := sha256.Sum256(raw) |
| + if digests[ref] != hex.EncodeToString(sum[:]) { |
| + return out, ErrIdentity |
| + } |
| + } |
| + var envelope CensusEnvelope |
| + if decode(raw, &envelope) != nil { |
| + return out, ErrIdentity |
| + } |
| + report := envelope.Report |
| + canonical, e := json.Marshal(report) |
| + if e != nil || len(box.CensusPublicKey) != ed25519.PublicKeySize || !ed25519.Verify(box.CensusPublicKey, canonical, envelope.Signature) { |
| + return out, ErrIdentity |
| + } |
| + if tag == "" || !report.Complete || report.Box != ref || report.Enrollment != box.Enrollment || report.Reservation != reservation || report.DirectoryRevision != roster.Roster.Revision || report.BootID == "" || report.CapturedAt.IsZero() || report.Installations == nil { |
| + return out, ErrIdentity |
| + } |
| + seen := map[string]bool{} |
| + for _, i := range report.Installations { |
| + if i.Host == "" || seen[i.Host] || (!i.Route && !i.Local) { |
| + return out, ErrIdentity |
| + } |
| + seen[i.Host] = true |
| + r, ok := allRecords[i.Host] |
| + if !ok || i.Account != r.Holder.Account || i.App != r.Holder.App || i.Key != r.Holder.Key || i.Family != r.Holder.Family { |
| + return out, ErrIdentity |
| + } |
| + inc, err := r.Incarnation() |
| + if err != nil || (i.Incarnation != "" && i.Incarnation != inc) { |
| + return out, ErrIdentity |
| + } |
| + } |
| + out.Reports[ref] = report |
| + } |
| + if digests != nil && len(digests) != len(out.Reports) { |
| + return out, ErrIdentity |
| + } |
| + fresh, err := a.Directory.Read(ctx) |
| + if err != nil { |
| + return out, err |
| + } |
| + if fresh.ETag != roster.ETag { |
| + return out, ErrPending |
| + } |
| + return out, nil |
| +} |
| +func (a ProductionAuthority) Census(ctx context.Context, account, app, reservation string) (Census, error) { |
| + return a.census(ctx, account, app, reservation, 0, nil) |
| +} |
| +func (a ProductionAuthority) Inventory(ctx context.Context, obs hostclaim.Observed) ([]string, []string, error) { |
| + roster, err := a.Directory.Read(ctx) |
| + if err != nil { |
| + return nil, nil, err |
| + } |
| + if len(roster.Roster.Regions) == 0 { |
| + return nil, nil, ErrContract |
| + } |
| + regions := make([]string, 0, len(roster.Roster.Regions)) |
| + boxes := make([]string, 0, len(roster.Roster.Boxes)) |
| + for r := range roster.Roster.Regions { |
| + regions = append(regions, r) |
| + } |
| + for b := range roster.Roster.Boxes { |
| + boxes = append(boxes, b) |
| + } |
| + sort.Strings(regions) |
| + sort.Strings(boxes) |
| + return regions, boxes, nil |
| +} |
| +func (a ProductionAuthority) Adoption(ctx context.Context, r Record) (region, box string, regions, boxes []string, err error) { |
| + if a.Backfill == nil { |
| + err = ErrContract |
| + return |
| + } |
| + return a.Backfill.Adoption(ctx, r) |
| +} |
| +func (b *Backfill) Adoption(ctx context.Context, r Record) (region, box string, regions, boxes []string, err error) { |
| + if err = CheckHostQuarantine(ctx, b.Store, r.Holder.Host); err != nil { |
| + return |
| + } |
| + p, tag, e := b.read(ctx) |
| + if e != nil { |
| + err = e |
| + return |
| + } |
| + selected, ok := p.Hosts[r.Holder.Host] |
| + if !ok || selected.Key != r.Holder.Key || selected.ETag != r.ETag || selected.Acquisition == "" || selected.Region == "" || selected.Box == "" { |
| + err = ErrIdentity |
| + return |
| + } |
| + a := ProductionAuthority{Store: b.Store, Directory: b.Directory} |
| + b.mu.Lock() |
| + if b.cachedTag != tag { |
| + b.cached = nil |
| + } |
| + var census Census |
| + if b.cached != nil { |
| + census = *b.cached |
| + } else { |
| + census, e = a.census(ctx, "", "", p.ID, p.DirectoryRevision, p.Reports) |
| + if e == nil { |
| + b.cached = &census |
| + b.cachedTag = tag |
| + } |
| + } |
| + b.mu.Unlock() |
| + if e != nil { |
| + err = e |
| + return |
| + } |
| + current, e := a.Directory.Read(ctx) |
| + if e != nil { |
| + err = e |
| + return |
| + } |
| + if current.ETag != census.Roster.ETag { |
| + err = ErrPending |
| + return |
| + } |
| + owners := map[string]bool{} |
| + found := false |
| + for _, rec := range census.Records { |
| + if rec.Holder.Host == r.Holder.Host { |
| + found = rec.ETag == r.ETag && rec.Holder.Key == r.Holder.Key |
| + } |
| + } |
| + for ref, report := range census.Reports { |
| + for _, i := range report.Installations { |
| + if i.Host == r.Holder.Host { |
| + owners[ref] = true |
| + } |
| + } |
| + } |
| + if !found || len(owners) != 1 || !owners[selected.Box] { |
| + err = ErrIdentity |
| + return |
| + } |
| + enrolled, ok := census.Roster.Roster.Boxes[selected.Box] |
| + if !ok || enrolled.Region != selected.Region || enrolled.State != "active" { |
| + err = ErrIdentity |
| + return |
| + } |
| + if r.Holder.Family == "instance" { |
| + var instance provision.Record |
| + if json.Unmarshal(r.Body, &instance) != nil || len(instance.Replicas) != 0 || provision.ClaimEverywhere(instance) { |
| + err = ErrIdentity |
| + return |
| + } |
| + } |
| + eligible := 0 |
| + for _, candidate := range census.Roster.Roster.Boxes { |
| + if candidate.State == "active" && candidate.Region == selected.Region && candidate.Namespace == enrolled.Namespace && candidate.Pool == enrolled.Pool { |
| + eligible++ |
| + } |
| + } |
| + if eligible != 1 { |
| + err = ErrContract |
| + return |
| + } |
| + raw, _, e := b.Store.Get(ctx, domains.ClaimKey(r.Holder.Host)) |
| + if e != nil { |
| + err = e |
| + return |
| + } |
| + var ownership domains.Claim |
| + inc, e := r.Incarnation() |
| + if e != nil { |
| + err = e |
| + return |
| + } |
| + if decode(raw, &ownership) != nil || ownership.Host != r.Holder.Host || ownership.Account != r.Holder.Account || ownership.App != r.Holder.App || ownership.Acquisition != selected.Acquisition || ownership.RecordIncarnation != inc || ownership.OwnershipState != "committed" || ownership.Pending { |
| + err = ErrIdentity |
| + return |
| + } |
| + if r.Holder.Acquisition != selected.Acquisition { |
| + err = ErrIdentity |
| + return |
| + } |
| + _, freshTag, e := b.read(ctx) |
| + if e != nil { |
| + err = e |
| + return |
| + } |
| + if freshTag != tag { |
| + err = ErrPending |
| + return |
| + } |
| + for region := range census.Roster.Roster.Regions { |
| + if region != selected.Region { |
| + regions = append(regions, region) |
| + } |
| + } |
| + for ref := range census.Roster.Roster.Boxes { |
| + if ref != selected.Box { |
| + boxes = append(boxes, ref) |
| + } |
| + } |
| + sort.Strings(regions) |
| + sort.Strings(boxes) |
| + return selected.Region, selected.Box, regions, boxes, nil |
| +} |
diff --git a/internal/hostops/backfill_controller.go b/internal/hostops/backfill_controller.go| new file mode 100644 |
| index 000000000..f6dcbe5ca |
| --- /dev/null |
| +++ b/internal/hostops/backfill_controller.go |
| @@ -0,0 +1,119 @@ |
| +package hostops |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// BackfillAuthority deliberately grants only the frozen backfill's adoption. |
| +// Mutation doors use ProductionAuthority once their full construction lands. |
| +type BackfillAuthority struct{ p ProductionAuthority } |
| + |
| +func (a BackfillAuthority) Inventory(ctx context.Context, obs hostclaim.Observed) ([]string, []string, error) { |
| + return a.p.Inventory(ctx, obs) |
| +} |
| +func (a BackfillAuthority) Adoption(ctx context.Context, r Record) (string, string, []string, []string, error) { |
| + return a.p.Adoption(ctx, r) |
| +} |
| +func (a BackfillAuthority) ClaimBackfill() *Backfill { return a.p.Backfill } |
| + |
| +func (a BackfillAuthority) Create(context.Context, hostclaim.Observed, Record, string) error { |
| + return ErrContract |
| +} |
| +func (a BackfillAuthority) Executor(context.Context, hostclaim.Observed, Record) (string, error) { |
| + return "", ErrContract |
| +} |
| +func (a BackfillAuthority) StartDelete(context.Context, hostclaim.Observed, Record) error { |
| + return ErrContract |
| +} |
| +func (a BackfillAuthority) FirstPhase(context.Context, hostclaim.Observed) error { return ErrContract } |
| +func (a BackfillAuthority) PurgeRetirement(context.Context, hostclaim.Observed) error { |
| + return ErrContract |
| +} |
| +func NewBackfillController(s release.Store, d hostactors.Directory, now func() time.Time, newID func() string) Controller { |
| + b := &Backfill{Store: s, Directory: d} |
| + return Controller{Store: s, Now: now, NewID: newID, Authority: BackfillAuthority{ProductionAuthority{Store: s, Directory: d, Backfill: b, Now: now}}} |
| +} |
| + |
| +func ReconcileClaims(ctx context.Context, c Controller, adopt func(context.Context, string) (hostclaim.Observed, error)) error { |
| + if adopt == nil { |
| + return ErrContract |
| + } |
| + var backfill *Backfill |
| + if provider, ok := c.Authority.(interface{ ClaimBackfill() *Backfill }); ok { |
| + backfill = provider.ClaimBackfill() |
| + } |
| + if backfill != nil { |
| + if _, _, err := backfill.read(ctx); err != nil { |
| + if errors.Is(err, os.ErrNotExist) { |
| + return ErrPending |
| + } |
| + return err |
| + } |
| + } |
| + apps, err := Apps(ctx, c.Store) |
| + if err != nil { |
| + return err |
| + } |
| + var all []error |
| + // Collect the entire listing before assigning any hostname. A broken page |
| + // cannot authorize an adoption against only the prefix read so far. |
| + var records []Record |
| + seen := map[string]bool{} |
| + for _, app := range apps { |
| + rs, err := Records(ctx, c.Store, app.Account, app.Name) |
| + if err != nil { |
| + return err |
| + } |
| + for _, r := range rs { |
| + if seen[r.Holder.Host] { |
| + return ErrIdentity |
| + } |
| + seen[r.Holder.Host] = true |
| + records = append(records, r) |
| + } |
| + } |
| + for _, listed := range records { |
| + r, err := ReadRecord(ctx, c.Store, listed.Holder.Key) |
| + if errors.Is(err, os.ErrNotExist) { |
| + continue |
| + } |
| + if err != nil { |
| + all = append(all, err) |
| + continue |
| + } |
| + if err = CheckHostQuarantine(ctx, c.Store, r.Holder.Host); err != nil { |
| + all = append(all, err) |
| + continue |
| + } |
| + obs, err := hostclaim.Read(ctx, c.Store, r.Holder.Host) |
| + if err != nil { |
| + all = append(all, err) |
| + continue |
| + } |
| + if !obs.Absent && (obs.Claim.Holder == nil || obs.Claim.Holder.Acquisition != "" || r.Holder.Acquisition == "") { |
| + continue |
| + } |
| + if _, err = adopt(ctx, r.Holder.Key); err != nil { |
| + all = append(all, err) |
| + } |
| + } |
| + return errors.Join(all...) |
| +} |
| +func (c Controller) ReconcileClaims(ctx context.Context) error { |
| + return ReconcileClaims(ctx, c, func(ctx context.Context, key string) (hostclaim.Observed, error) { |
| + rec, err := ReadRecord(ctx, c.Store, key) |
| + if err != nil { |
| + return hostclaim.Observed{}, err |
| + } |
| + return c.Adopt(ctx, rec) |
| + }) |
| +} |
| + |
| +func (a ProductionAuthority) ClaimBackfill() *Backfill { return a.Backfill } |
diff --git a/internal/hostops/backfill_publish.go b/internal/hostops/backfill_publish.go| new file mode 100644 |
| index 000000000..9f770662c |
| --- /dev/null |
| +++ b/internal/hostops/backfill_publish.go |
| @@ -0,0 +1,162 @@ |
| +package hostops |
| + |
| +import ( |
| + "bytes" |
| + "context" |
| + "crypto/ed25519" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "errors" |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Publish freezes exact report bytes into #232's already-fenced backfill. |
| +// The collecting manifest is the global create/record-writer freeze. Its ETag |
| +// is supplied by the #232 coordinator, which retains that freeze through done. |
| +func (b *Backfill) Publish(ctx context.Context, signer release.Signer, collectingETag string) error { |
| + raw, tag, err := b.Store.Get(ctx, BackfillKey) |
| + if err != nil { |
| + return err |
| + } |
| + var p BackfillPlan |
| + if tag != collectingETag || tag == "" || release.VerifyRootEnvelope(b.Directory.PublicKey, raw, &p) != nil || p.V != 1 || p.Phase != "collecting" || p.ID == "" || p.CreatedAt.IsZero() || p.Hosts == nil { |
| + return ErrContract |
| + } |
| + a := ProductionAuthority{Store: b.Store, Directory: b.Directory} |
| + census, err := a.census(ctx, "", "", p.ID, p.DirectoryRevision, nil) |
| + if err != nil { |
| + return err |
| + } |
| + p.Reports = map[string]string{} |
| + for box, report := range census.Reports { |
| + if report.CapturedAt.Before(p.CreatedAt) { |
| + return ErrIdentity |
| + } |
| + key, err := reportKey(p.ID, box) |
| + if err != nil { |
| + return err |
| + } |
| + body, _, err := b.Store.Get(ctx, key) |
| + if err != nil { |
| + return err |
| + } |
| + var envelope CensusEnvelope |
| + if json.Unmarshal(body, &envelope) != nil || envelope.Report.Reservation != report.Reservation || !envelope.Report.Complete { |
| + return ErrIdentity |
| + } |
| + sum := sha256.Sum256(body) |
| + p.Reports[box] = hex.EncodeToString(sum[:]) |
| + } |
| + for _, r := range census.Records { |
| + if err := CheckHostQuarantine(ctx, b.Store, r.Holder.Host); err != nil { |
| + return err |
| + } |
| + selected, ok := p.Hosts[r.Holder.Host] |
| + if !ok || selected.Key != r.Holder.Key || selected.ETag != r.ETag || selected.Acquisition == "" || r.Holder.Acquisition != selected.Acquisition { |
| + return ErrIdentity |
| + } |
| + } |
| + if len(p.Hosts) != len(census.Records) { |
| + return ErrIdentity |
| + } |
| + p.Phase = "adopting" |
| + body, err := release.SignRootEnvelope(signer, p) |
| + if err != nil { |
| + return err |
| + } |
| + _, err = b.Store.Put(ctx, BackfillKey, body, tag) |
| + return err |
| +} |
| + |
| +// PublishCensusReport is called by the guarded box census collector. Each box |
| +// owns its private signing key; the root-signed roster binds its public key to |
| +// verified enrollment. The console has only that public key. |
| +func PublishCensusReport(ctx context.Context, store release.Store, d hostactors.Directory, report CensusReport, private ed25519.PrivateKey) error { |
| + if len(private) != ed25519.PrivateKeySize { |
| + return ErrContract |
| + } |
| + roster, err := d.Read(ctx) |
| + if err != nil { |
| + return err |
| + } |
| + box, ok := roster.Roster.Boxes[report.Box] |
| + if !ok || report.Enrollment != box.Enrollment || report.DirectoryRevision != roster.Roster.Revision || !bytes.Equal(private.Public().(ed25519.PublicKey), box.CensusPublicKey) || report.BootID == "" || !report.Complete || report.Installations == nil || report.CapturedAt.IsZero() { |
| + return ErrIdentity |
| + } |
| + canonical, err := json.Marshal(report) |
| + if err != nil { |
| + return err |
| + } |
| + raw, err := json.Marshal(CensusEnvelope{Report: report, Signature: ed25519.Sign(private, canonical)}) |
| + if err != nil { |
| + return err |
| + } |
| + key, err := reportKey(report.Reservation, report.Box) |
| + if err != nil { |
| + return err |
| + } |
| + if _, err = store.Put(ctx, key, raw, ""); err == nil { |
| + return nil |
| + } |
| + if !errors.Is(err, release.ErrCASConflict) { |
| + return err |
| + } |
| + previous, _, err := store.Get(ctx, key) |
| + if err != nil { |
| + return err |
| + } |
| + if !bytes.Equal(previous, raw) { |
| + return ErrIdentity |
| + } |
| + return nil |
| +} |
| + |
| +// AuthorizeBackfillApp is the only legacy app-admission initialization proof. |
| +// It accepts the same complete frozen census as host adoption, including apps |
| +// with no host records. Empty results from a failed listing never qualify. |
| +func (a ProductionAuthority) AuthorizeBackfillApp(ctx context.Context, account, app string) (string, error) { |
| + if a.Backfill == nil || !safeSegment(account) || !safeSegment(app) { |
| + return "", ErrContract |
| + } |
| + p, tag, err := a.Backfill.read(ctx) |
| + if err != nil { |
| + return "", err |
| + } |
| + census, err := a.census(ctx, account, app, p.ID, p.DirectoryRevision, p.Reports) |
| + if err != nil { |
| + return "", err |
| + } |
| + apps, err := Apps(ctx, a.Store) |
| + if err != nil { |
| + return "", err |
| + } |
| + found := false |
| + for _, candidate := range apps { |
| + if candidate.Account == account && candidate.Name == app { |
| + found = true |
| + } |
| + } |
| + if !found { |
| + return "", ErrIdentity |
| + } |
| + for _, r := range census.Records { |
| + planned, ok := p.Hosts[r.Holder.Host] |
| + if !ok || planned.Key != r.Holder.Key || planned.ETag != r.ETag || planned.Acquisition == "" || planned.Acquisition != r.Holder.Acquisition { |
| + return "", ErrIdentity |
| + } |
| + if err = CheckHostQuarantine(ctx, a.Store, r.Holder.Host); err != nil { |
| + return "", err |
| + } |
| + } |
| + _, fresh, err := a.Backfill.read(ctx) |
| + if err != nil { |
| + return "", err |
| + } |
| + if fresh != tag { |
| + return "", ErrPending |
| + } |
| + return p.ID, nil |
| +} |
diff --git a/internal/hostops/backfill_test.go b/internal/hostops/backfill_test.go| new file mode 100644 |
| index 000000000..878cb53a2 |
| --- /dev/null |
| +++ b/internal/hostops/backfill_test.go |
| @@ -0,0 +1,233 @@ |
| +package hostops |
| + |
| +import ( |
| + "context" |
| + "crypto/ed25519" |
| + "crypto/rand" |
| + "crypto/sha256" |
| + "encoding/hex" |
| + "encoding/json" |
| + "errors" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/domains" |
| + "github.com/carlosframework/platform/internal/hostactors" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +func TestReconcileFrozenBackfillAndQuarantine(t *testing.T) { |
| + ctx := context.Background() |
| + f := newFixture(t) |
| + pub, key, err := ed25519.GenerateKey(rand.Reader) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + sg := release.RootSigner(key) |
| + roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Regions: map[string]hostactors.Region{"ie": {}, "us": {}}, Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}} |
| + boxKeys := map[string]ed25519.PrivateKey{} |
| + for i, ref := range []string{"box-a", "box-b"} { |
| + boxPub, boxKey, e := ed25519.GenerateKey(rand.Reader) |
| + if e != nil { |
| + t.Fatal(e) |
| + } |
| + boxKeys[ref] = boxKey |
| + region := "ie" |
| + en := strings.Repeat("a", 32) |
| + if i == 1 { |
| + region = "us" |
| + en = strings.Repeat("b", 32) |
| + } |
| + roster.Boxes[ref] = hostactors.Box{CensusPublicKey: boxPub, Enrollment: en, Namespace: "platform", BoxID: ref, Label: ref, Region: region, Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: region, InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"} |
| + } |
| + put := func(k string, v any, signed bool) string { |
| + t.Helper() |
| + var raw []byte |
| + var e error |
| + if signed { |
| + raw, e = release.SignRootEnvelope(sg, v) |
| + } else { |
| + raw, e = json.Marshal(v) |
| + } |
| + if e != nil { |
| + t.Fatal(e) |
| + } |
| + tag, e := f.store.Put(ctx, k, raw, "") |
| + if e != nil { |
| + t.Fatal(e) |
| + } |
| + return tag |
| + } |
| + put(hostactors.Key, roster, true) |
| + d := hostactors.Directory{Store: f.store, PublicKey: pub, Bucket: "bucket"} |
| + c := NewBackfillController(f.store, d, func() time.Time { return f.now }, f.c.NewID) |
| + r := f.record(t, "legacy.example.com") |
| + var body map[string]any |
| + if err = json.Unmarshal(r.Body, &body); err != nil { |
| + t.Fatal(err) |
| + } |
| + body["acq"] = "ownership-acquisition" |
| + put(r.Holder.Key, body, false) |
| + r, err = ReadRecord(ctx, f.store, r.Holder.Key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + inc, err := r.Incarnation() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + put(domains.ClaimKey(r.Holder.Host), domains.Claim{Schema: 1, Host: r.Holder.Host, Account: "bab", App: "jam", Acquisition: "ownership-acquisition", RecordIncarnation: inc, OwnershipState: "committed"}, false) |
| + if err = c.ReconcileClaims(ctx); !errors.Is(err, ErrPending) { |
| + t.Fatalf("unfrozen adoption: %v", err) |
| + } |
| + obs, err := hostclaim.Read(ctx, f.store, r.Holder.Host) |
| + if err != nil || !obs.Absent { |
| + t.Fatal(obs, err) |
| + } |
| + p := BackfillPlan{V: 1, ID: "backfill", Phase: "adopting", DirectoryRevision: 1, CreatedAt: f.now, Reports: map[string]string{}, Hosts: map[string]BackfillHost{r.Holder.Host: {Key: r.Holder.Key, ETag: r.ETag, Acquisition: "ownership-acquisition", Region: "ie", Box: "box-a"}}} |
| + for ref, b := range roster.Boxes { |
| + report := CensusReport{Box: ref, Enrollment: b.Enrollment, BootID: "boot", Reservation: p.ID, DirectoryRevision: 1, CapturedAt: f.now, Complete: true, Installations: []CensusInstallation{}} |
| + if ref == "box-a" { |
| + report.Installations = append(report.Installations, CensusInstallation{Host: r.Holder.Host, Account: "bab", App: "jam", Family: "instance", Key: r.Holder.Key, Incarnation: inc, Local: true, Route: true}) |
| + } |
| + key, _ := reportKey(p.ID, ref) |
| + body, e := json.Marshal(report) |
| + if e != nil { |
| + t.Fatal(e) |
| + } |
| + put(key, CensusEnvelope{Report: report, Signature: ed25519.Sign(boxKeys[ref], body)}, false) |
| + raw, _, err := f.store.Get(ctx, key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + sum := sha256.Sum256(raw) |
| + p.Reports[ref] = hex.EncodeToString(sum[:]) |
| + } |
| + put(BackfillKey, p, true) |
| + qtag := put(HostsQuarantineKey, HostsQuarantine{V: 1, Hosts: map[string]string{r.Holder.Host: "conflict"}}, false) |
| + if err = c.ReconcileClaims(ctx); !errors.Is(err, ErrPending) { |
| + t.Fatalf("quarantine: %v", err) |
| + } |
| + raw, _ := json.Marshal(HostsQuarantine{V: 1, Hosts: map[string]string{}}) |
| + if _, err = f.store.Put(ctx, HostsQuarantineKey, raw, qtag); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + // A digest listed by a root manifest cannot turn a forged box report into |
| + // an authenticated census. Test the verifier before its cache is populated. |
| + reportPath, _ := reportKey(p.ID, "box-a") |
| + authentic, reportTag, e := f.store.Get(ctx, reportPath) |
| + if e != nil { |
| + t.Fatal(e) |
| + } |
| + var forged CensusEnvelope |
| + if e = json.Unmarshal(authentic, &forged); e != nil { |
| + t.Fatal(e) |
| + } |
| + forged.Signature[0] ^= 1 |
| + forgedRaw, _ := json.Marshal(forged) |
| + forgedTag, e := f.store.Put(ctx, reportPath, forgedRaw, reportTag) |
| + if e != nil { |
| + t.Fatal(e) |
| + } |
| + sum := sha256.Sum256(forgedRaw) |
| + p.Reports["box-a"] = hex.EncodeToString(sum[:]) |
| + _, manifestTag, e := f.store.Get(ctx, BackfillKey) |
| + if e != nil { |
| + t.Fatal(e) |
| + } |
| + signed, _ := release.SignRootEnvelope(sg, p) |
| + manifestTag, e = f.store.Put(ctx, BackfillKey, signed, manifestTag) |
| + if e != nil { |
| + t.Fatal(e) |
| + } |
| + if e = c.ReconcileClaims(ctx); e == nil { |
| + t.Fatal("accepted forged census") |
| + } |
| + if _, e = f.store.Put(ctx, reportPath, authentic, forgedTag); e != nil { |
| + t.Fatal(e) |
| + } |
| + sum = sha256.Sum256(authentic) |
| + p.Reports["box-a"] = hex.EncodeToString(sum[:]) |
| + signed, _ = release.SignRootEnvelope(sg, p) |
| + if _, e = f.store.Put(ctx, BackfillKey, signed, manifestTag); e != nil { |
| + t.Fatal(e) |
| + } |
| + if err = c.ReconcileClaims(ctx); err != nil { |
| + t.Fatal(err) |
| + } |
| + obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if obs.Absent || obs.Claim.Holder.Acquisition != "ownership-acquisition" || obs.Claim.Op.Exec != "box-a" || obs.Claim.Touched.Complete || len(obs.Claim.Obligations) != 2 { |
| + t.Fatalf("bad backfill: %+v", obs.Claim) |
| + } |
| + before := obs.ETag |
| + if err = c.ReconcileClaims(ctx); err != nil { |
| + t.Fatal(err) |
| + } |
| + obs, err = hostclaim.Read(ctx, f.store, r.Holder.Host) |
| + if err != nil || obs.ETag != before { |
| + t.Fatal("non-idempotent", err) |
| + } |
| +} |
| +func TestReconcileDiscardPartialCensus(t *testing.T) { |
| + f := newFixture(t) |
| + r := f.record(t, "legacy.example.com") |
| + if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + fail := errors.New("second listing failed") |
| + f.store.afterList = func(prefix string, keys []string) error { |
| + if strings.Contains(prefix, "sidecars/") { |
| + return fail |
| + } |
| + return nil |
| + } |
| + calls := 0 |
| + err := ReconcileClaims(f.ctx, f.c, func(context.Context, string) (hostclaim.Observed, error) { calls++; return hostclaim.Observed{}, nil }) |
| + if !errors.Is(err, fail) || calls != 0 { |
| + t.Fatal(err, calls) |
| + } |
| +} |
| + |
| +func TestClaimDeleteDoorNeverAdopts(t *testing.T) { |
| + f := newFixture(t) |
| + r := f.record(t, "legacy.example.com") |
| + if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + inc, err := r.Incarnation() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err = f.c.BeginRetire(f.ctx, r.Holder.Key, "test", inc); !errors.Is(err, ErrPending) { |
| + t.Fatal(err) |
| + } |
| + obs, err := hostclaim.Read(f.ctx, f.store, r.Holder.Host) |
| + if err != nil || !obs.Absent { |
| + t.Fatalf("door adopted: %+v %v", obs, err) |
| + } |
| +} |
| + |
| +func TestBackfillAuthorityCannotGainMutationMethods(t *testing.T) { |
| + c := NewBackfillController(nil, hostactors.Directory{}, time.Now, func() string { return "id" }) |
| + if _, ok := c.Authority.(interface { |
| + StartCreate(context.Context, hostclaim.Observed, Record, string, string) (hostclaim.Observed, error) |
| + }); ok { |
| + t.Fatal("backfill gained create authority") |
| + } |
| + if _, ok := c.Authority.(interface { |
| + StartDeleteClaim(context.Context, hostclaim.Observed, Record, string) (hostclaim.Observed, error) |
| + }); ok { |
| + t.Fatal("backfill gained delete authority") |
| + } |
| + if _, ok := c.Authority.(interface { |
| + RecordEffects(context.Context, hostclaim.Observed, Record) error |
| + }); ok { |
| + t.Fatal("backfill gained effect authority") |
| + } |
| +} |
diff --git a/internal/hostops/operations.go b/internal/hostops/operations.go| index 4409ef9b5..aea6021d2 100644 |
| --- a/internal/hostops/operations.go |
| +++ b/internal/hostops/operations.go |
| @@ -142,10 +142,29 @@ func (c Controller) Adopt(ctx context.Context, r Record) (hostclaim.Observed, er |
| return obs, err |
| } |
| if !obs.Absent { |
| - if !matches(r, obs.Claim.Holder) { |
| + if matches(r, obs.Claim.Holder) { |
| + return obs, nil |
| + } |
| + if obs.Claim.Holder == nil || obs.Claim.Holder.Acquisition != "" || r.Holder.Acquisition == "" { |
| + return obs, ErrIdentity |
| + } |
| + // #232 also repairs already-claimed legacy holders while globally |
| + // frozen. Adoption authority verifies the exact manifest/ownership. |
| + prior := *obs.Claim.Holder |
| + prior.Acquisition = r.Holder.Acquisition |
| + if !matches(r, &prior) { |
| return obs, ErrIdentity |
| } |
| - return obs, nil |
| + if _, _, _, _, err := c.Authority.Adoption(ctx, r); err != nil { |
| + return obs, err |
| + } |
| + return hostclaim.Update(ctx, c.Store, obs, func(next *hostclaim.Claim) error { |
| + if next.Holder == nil || next.Holder.Acquisition != "" { |
| + return ErrPending |
| + } |
| + next.Holder.Acquisition = r.Holder.Acquisition |
| + return nil |
| + }) |
| } |
| region, box, regions, boxes, err := c.Authority.Adoption(ctx, r) |
| if err != nil { |
| @@ -181,10 +200,16 @@ func (c Controller) BeginRetire(ctx context.Context, key, by, expectedInc string |
| if expectedInc == "" || inc != expectedInc { |
| return hostclaim.Observed{}, ErrIdentity |
| } |
| - obs, err := c.Adopt(ctx, r) |
| + obs, err := hostclaim.Read(ctx, c.Store, r.Holder.Host) |
| if err != nil { |
| return obs, err |
| } |
| + if obs.Absent { |
| + return obs, ErrPending |
| + } |
| + if !matches(r, obs.Claim.Holder) { |
| + return obs, ErrIdentity |
| + } |
| if err := c.Authority.StartDelete(ctx, obs, r); err != nil { |
| return obs, err |
| } |
diff --git a/internal/hostops/record.go b/internal/hostops/record.go| index 7616db1b0..25223712f 100644 |
| --- a/internal/hostops/record.go |
| +++ b/internal/hostops/record.go |
| @@ -51,7 +51,13 @@ func ReadRecord(ctx context.Context, s release.Store, key string) (Record, error |
| if etag == "" { |
| return Record{}, ErrIdentity |
| } |
| - n := hostclaim.NewHolder{Key: key, Account: parts[1], App: parts[2]} |
| + var binding struct { |
| + Acquisition string `json:"acq"` |
| + } |
| + if err := decode(body, &binding); err != nil { |
| + return Record{}, err |
| + } |
| + n := hostclaim.NewHolder{Acquisition: binding.Acquisition, Key: key, Account: parts[1], App: parts[2]} |
| switch parts[3] { |
| case "instances": |
| var r provision.Record |
| @@ -92,11 +98,14 @@ func matches(r Record, h *hostclaim.Holder) bool { |
| } |
| n := r.Holder |
| inc, err := r.Incarnation() |
| - return err == nil && n.Key == h.Key && n.Family == h.Family && n.Account == h.Account && n.App == h.App && n.Name == h.Name && n.CreatedAt.Equal(h.CreatedAt) && inc == h.Incarnation |
| + return err == nil && n.Acquisition == h.Acquisition && n.Key == h.Key && n.Family == h.Family && n.Account == h.Account && n.App == h.App && n.Name == h.Name && n.CreatedAt.Equal(h.CreatedAt) && inc == h.Incarnation |
| } |
| // The caller reads the record first. This function performs the later claim read. |
| func RemoveOrphan(ctx context.Context, s release.Store, r Record) (bool, error) { |
| + if err := CheckHostQuarantine(ctx, s, r.Holder.Host); err != nil { |
| + return false, err |
| + } |
| inc, err := r.Incarnation() |
| if err != nil { |
| return false, err |
| @@ -155,7 +164,7 @@ func PrepareInstance(r provision.Record, now time.Time, region string) (Record, |
| if err != nil { |
| return Record{}, err |
| } |
| - n := hostclaim.NewHolder{Family: "instance", Key: provision.RecordKey(r.Account, r.App, r.Host), Account: r.Account, App: r.App, Host: r.Host, RecordRegion: r.Region, Region: region, CreatedAt: r.CreatedAt} |
| + n := hostclaim.NewHolder{Acquisition: r.Acquisition, Family: "instance", Key: provision.RecordKey(r.Account, r.App, r.Host), Account: r.Account, App: r.App, Host: r.Host, RecordRegion: r.Region, Region: region, CreatedAt: r.CreatedAt} |
| return Record{Holder: n, Body: body}, nil |
| } |
| @@ -168,6 +177,6 @@ func PrepareSidecar(r provision.SidecarRecord, now time.Time, region string) (Re |
| if err != nil { |
| return Record{}, err |
| } |
| - n := hostclaim.NewHolder{Family: "sidecar", Key: provision.SidecarRecordKey(r.Account, r.App, r.Name), Account: r.Account, App: r.App, Name: r.Name, Host: r.Host, RecordRegion: r.Region, Region: region, CreatedAt: r.CreatedAt} |
| + n := hostclaim.NewHolder{Acquisition: r.Acquisition, Family: "sidecar", Key: provision.SidecarRecordKey(r.Account, r.App, r.Name), Account: r.Account, App: r.App, Name: r.Name, Host: r.Host, RecordRegion: r.Region, Region: region, CreatedAt: r.CreatedAt} |
| return Record{Holder: n, Body: body}, nil |
| } |
diff --git a/internal/hostops/recovery.go b/internal/hostops/recovery.go| index c134a69b1..26a9f2929 100644 |
| --- a/internal/hostops/recovery.go |
| +++ b/internal/hostops/recovery.go |
| @@ -178,7 +178,7 @@ func (c Controller) SweepAllRecords(ctx context.Context) error { |
| continue |
| } |
| if obs.Absent { |
| - _, err = c.Adopt(ctx, record) |
| + err = ErrPending |
| } else { |
| _, err = RemoveOrphan(ctx, c.Store, record) |
| } |
diff --git a/internal/hostops/retirement.go b/internal/hostops/retirement.go| index 573d30b40..f0ee2979d 100644 |
| --- a/internal/hostops/retirement.go |
| +++ b/internal/hostops/retirement.go |
| @@ -29,10 +29,7 @@ func (c Controller) RetireRecord(ctx context.Context, ref provision.RetireRef) e |
| if errors.Is(rerr, os.ErrNotExist) { |
| return nil |
| } |
| - obs, err = c.Adopt(ctx, rec) |
| - if err != nil { |
| - return err |
| - } |
| + return ErrPending |
| } |
| h := obs.Claim.Holder |
| if h == nil { |
diff --git a/internal/provision/claim_export.go b/internal/provision/claim_export.go| new file mode 100644 |
| index 000000000..3a41dc9e6 |
| --- /dev/null |
| +++ b/internal/provision/claim_export.go |
| @@ -0,0 +1,3 @@ |
| +package provision |
| + |
| +func ClaimEverywhere(rec Record) bool { return everywhereClass(rec) } |
diff --git a/internal/provision/provision.go b/internal/provision/provision.go| index 2616cdfda..5d1f0b311 100644 |
| --- a/internal/provision/provision.go |
| +++ b/internal/provision/provision.go |
| @@ -37,6 +37,7 @@ import ( |
| // Account vs the key's segments); GetRecord and ListRecords enforce that, a |
| // record that lies about identity is never guessed at (design §3.2). |
| type Record struct { |
| + Acquisition string `json:"acq,omitempty"` |
| // Maintenance is preserved by ordinary writers; only a future guarded |
| // maintenance transition may install, replace, or clear it. |
| Maintenance MaintenanceProjection `json:"maintenance,omitempty"` |
diff --git a/internal/provision/sidecar.go b/internal/provision/sidecar.go| index d6891ea85..f49fbaa4f 100644 |
| --- a/internal/provision/sidecar.go |
| +++ b/internal/provision/sidecar.go |
| @@ -69,6 +69,7 @@ type SidecarManifest struct { |
| // identity is never guessed at, the same discipline GetRecord applies to |
| // instances. |
| type SidecarRecord struct { |
| + Acquisition string `json:"acq,omitempty"` |
| Schema int `json:"schema"` // 1 |
| Name string `json:"name"` |
| Host string `json:"host"` |