amadan

carlos / platform Public

hostresolve, console, cli: operator resolution and readmission wired to the roster (claim Plan 8b Task 5)

Co-Authored-By: Codex <noreply@openai.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
codex pushed by claude 71e560643409513ce32ea8334c8c8d573525a61c parent 264381e
13 files changed, +1624 −27
  • cmd/console/main.go +2 −0
  • hostclaim_cli.go +67 −5
  • internal/console/apihostclaim.go +89 −19
  • internal/console/apihostclaim_test.go +1 −1
  • internal/console/apisystemupdate.go +2 −2
  • internal/console/hostactivation.go +227 −0
  • internal/console/hostactivation_test.go +207 −0
  • internal/console/hostreports.go +83 −0
  • internal/console/server.go +2 −0
  • internal/hostresolve/readmit.go +131 −0
  • internal/hostresolve/review3_test.go +243 −0
  • internal/hostresolve/service.go +318 −0
  • internal/hostresolve/service_test.go +252 −0
diff --git a/cmd/console/main.go b/cmd/console/main.go
index 804cfc5ad..693286c57 100644
--- a/cmd/console/main.go
+++ b/cmd/console/main.go
@@ -296,6 +296,7 @@ func serve() {
// The carlos db door's data bucket and this box's placement
// (dbDoorWiring).
HostClaim: hostClaim,
+ HostConsoleID: os.Getenv("CARLOS_CONSOLE_ID"),
DBVersions: dbVersions,
HibernateBucket: dbBucket,
HibernateRegion: dbRegion,
@@ -366,6 +367,7 @@ func serve() {
// operators on each fire/clear (internal/console/alertwatch.go). Started
// here rather than in NewServer so constructing a Server — which every
// console test does — never starts a goroutine or sends mail.
+ go srv.StartHostReports(context.Background())
go srv.StartAlertWatcher(context.Background())
// The senders migration's tail (tenant-email senders design §5): once a
// release has passed with every console writing senders/app.json, an
diff --git a/hostclaim_cli.go b/hostclaim_cli.go
index fce0098a8..0cf438948 100644
--- a/hostclaim_cli.go
+++ b/hostclaim_cli.go
@@ -1,19 +1,23 @@
package main
import (
+ "bufio"
"context"
+ "encoding/json"
"errors"
"fmt"
+ "github.com/carlosframework/platform/internal/awsinv"
+ "github.com/carlosframework/platform/internal/hostfence"
+ "github.com/carlosframework/platform/internal/hostresolve"
"log"
"net/url"
"os"
"strings"
+ "time"
boxops "github.com/carlosframework/platform/internal/ops"
)
-const hostClaimPendingRuling = "awaiting Paul's ruling P2/P3 (amadan #236)"
-
func hostClaimEnabled(raw string) bool { return strings.TrimSpace(raw) != "" }
type hostResolveOptions struct {
@@ -90,11 +94,9 @@ func runOpsHost(args []string) {
log.Fatal(err)
}
sess := requireDeployAPISession(o.Console)
- if err := sess.inspectHostResolve(context.Background(), o); err != nil {
+ if err := sess.resolveHost(context.Background(), o); err != nil {
log.Fatal(err)
}
- // the follow-on integration replaces this impossible branch when ActorDirectory exists.
- log.Fatal(hostClaimPendingRuling)
}
func (s *apiSession) inspectHostResolve(ctx context.Context, o hostResolveOptions) error {
@@ -109,3 +111,63 @@ func (s *apiSession) inspectHostResolve(ctx context.Context, o hostResolveOption
}
return nil
}
+
+func (s *apiSession) resolveHost(ctx context.Context, o hostResolveOptions) error {
+ base := "/api/cli/ops/hosts/" + url.PathEscape(o.Host) + "/resolve"
+ status, body, err := s.postJSON(ctx, base+"/inspect", map[string]any{"action": o.Action, "box": o.Box})
+ if err != nil {
+ return err
+ }
+ if status != 200 {
+ return apiErrorFromBody(status, body)
+ }
+ if o.Action == "readmit" {
+ var request map[string]any
+ if err = json.Unmarshal(body, &request); err != nil {
+ return err
+ }
+ fmt.Fprintln(os.Stdout, string(body))
+ if !o.Yes {
+ fmt.Fprint(os.Stdout, "Readmit this exact enrollment and revision? Type readmit: ")
+ line, _ := bufio.NewReader(os.Stdin).ReadString('\n')
+ if strings.TrimSpace(line) != "readmit" {
+ return errors.New("readmission not confirmed")
+ }
+ }
+ request["action"] = "readmit"
+ status, body, err = s.postJSON(ctx, base, request)
+ } else {
+ var inspection hostresolve.Inspection
+ if err = json.Unmarshal(body, &inspection); err != nil {
+ return err
+ }
+ c := inspection.Details
+ // TLS-authenticated console response; the console verifies its unchanged
+ // signed envelope on submission. The client does not call this AWS proof.
+ fmt.Fprintln(os.Stdout, string(body))
+ fmt.Fprintln(os.Stdout, "Confirm that compute remains fenced, any restart will be cold through quarantine, and the listed storage was destroyed if asserted. Type the declaration ID:")
+ line, _ := bufio.NewReader(os.Stdin).ReadString('\n')
+ if strings.TrimSpace(line) != c.ID {
+ return errors.New("fencing attestation not confirmed")
+ }
+ // --yes does not silently attest facts about compute or disks.
+ api, err := awsinv.NewEC2Reader(ctx, c.Cloud.Region)
+ if err != nil {
+ return err
+ }
+ observation, err := hostfence.ObserveForChallenge(ctx, api, c, time.Now)
+ if err != nil {
+ return err
+ }
+ submission := hostfence.Submission{Challenge: inspection.Challenge, Observation: observation, RequestID: observation.RequestID, Confirmation: c.ID, Attestation: hostfence.OperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, Disks: c.Consequences.DestroyedStorageAssertion}}
+ status, body, err = s.postJSON(ctx, base, map[string]any{"action": o.Action, "submission": submission})
+ }
+ if err != nil {
+ return err
+ }
+ if status != 200 {
+ return apiErrorFromBody(status, body)
+ }
+ fmt.Fprintln(os.Stdout, string(body))
+ return nil
+}
diff --git a/internal/console/apihostclaim.go b/internal/console/apihostclaim.go
index 399f10348..3788a0303 100644
--- a/internal/console/apihostclaim.go
+++ b/internal/console/apihostclaim.go
@@ -1,37 +1,107 @@
package console
import (
- "log"
+ "encoding/json"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostfence"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/hostresolve"
+ "github.com/carlosframework/platform/internal/release"
"net/http"
+ "time"
)
-const hostClaimPendingRuling = "awaiting Paul's ruling P2/P3 (amadan #236)"
+const hostClaimUnavailable = "host claim authority is unavailable"
-// hostClaimRefusal applies the D2 authority boundary before reporting that
-// the follow-on integration's ActorDirectory is unavailable. It deliberately performs no claim,
-// directory or cloud write.
-func (s *Server) hostClaimRefusal(w http.ResponseWriter, r *http.Request) {
+func (s *Server) hostClaimOperator(w http.ResponseWriter, r *http.Request) (string, bool) {
addr, ok := s.cliUser(w, r)
if !ok {
- return
+ return "", false
}
- isOp, err := s.store.IsOperator(addr)
- if err != nil {
- log.Printf("console: host claim operator lookup: %v", err)
+ yes, err := s.store.IsOperator(addr)
+ if err != nil || !yes || !s.cfg.HostClaim {
http.NotFound(w, r)
- return
+ return "", false
}
- if !isOp || !s.cfg.HostClaim {
- http.NotFound(w, r)
- return
+ if s.cfg.Bucket == "" || len(s.cfg.Key) == 0 || s.cfg.Releases == nil {
+ jsonError(w, http.StatusServiceUnavailable, hostClaimUnavailable)
+ return "", false
+ }
+ if _, err := hostfloor.Read(r.Context(), s.cfg.Releases, release.RootSigner(s.cfg.Key).RootPublicKey(), s.cfg.Bucket); err != nil {
+ jsonError(w, http.StatusServiceUnavailable, hostClaimUnavailable)
+ return "", false
}
- jsonError(w, http.StatusServiceUnavailable, hostClaimPendingRuling)
+ return addr, true
+}
+func (s *Server) hostResolver() hostresolve.Service {
+ signer := release.RootSigner(s.cfg.Key)
+ return hostresolve.Service{Directory: hostactors.Directory{Store: s.cfg.Releases, PublicKey: signer.RootPublicKey(), Bucket: s.cfg.Bucket}, Signer: signer, Now: time.Now}
}
-
func (s *Server) handleAPIHostResolveInspect(w http.ResponseWriter, r *http.Request) {
- s.hostClaimRefusal(w, r)
+ actor, ok := s.hostClaimOperator(w, r)
+ if !ok {
+ return
+ }
+ var in struct {
+ Action string `json:"action"`
+ Box string `json:"box"`
+ }
+ if json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&in) != nil {
+ jsonError(w, 400, "invalid resolution request")
+ return
+ }
+ svc := s.hostResolver()
+ if in.Action == "readmit" {
+ ref, b, err := svc.Directory.Resolve(r.Context(), in.Box)
+ if err != nil {
+ jsonError(w, 409, err.Error())
+ return
+ }
+ json.NewEncoder(w).Encode(map[string]any{"box": ref, "enrollment": b.Enrollment, "revision": b.QuarantineRevision, "loss_ids": b.LossIDs})
+ return
+ }
+ if in.Action != "declare-lost" {
+ jsonError(w, 400, "invalid resolution action")
+ return
+ }
+ inspected, err := svc.Inspect(r.Context(), actor, r.PathValue("host"), in.Box)
+ if err != nil {
+ jsonError(w, 409, err.Error())
+ return
+ }
+ json.NewEncoder(w).Encode(inspected)
}
-
func (s *Server) handleAPIHostResolve(w http.ResponseWriter, r *http.Request) {
- s.hostClaimRefusal(w, r)
+ actor, ok := s.hostClaimOperator(w, r)
+ if !ok {
+ return
+ }
+ var in struct {
+ Action string `json:"action"`
+ Submission hostfence.Submission `json:"submission"`
+ Box string `json:"box"`
+ Enrollment string `json:"enrollment"`
+ Revision uint64 `json:"revision"`
+ LossIDs []string `json:"loss_ids"`
+ }
+ if json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&in) != nil {
+ jsonError(w, 400, "invalid resolution request")
+ return
+ }
+ svc := s.hostResolver()
+ if in.Action == "readmit" {
+ if err := svc.Readmit(r.Context(), in.Box, in.Enrollment, in.Revision, in.LossIDs); err != nil {
+ jsonError(w, 409, err.Error())
+ return
+ }
+ } else if in.Action == "declare-lost" {
+ if _, err := svc.Declare(r.Context(), actor, r.PathValue("host"), in.Submission); err != nil {
+ jsonError(w, 409, err.Error())
+ return
+ }
+ } else {
+ jsonError(w, 400, "invalid resolution action")
+ return
+ }
+ json.NewEncoder(w).Encode(map[string]bool{"ok": true})
}
diff --git a/internal/console/apihostclaim_test.go b/internal/console/apihostclaim_test.go
index a5efcac67..3bab3afed 100644
--- a/internal/console/apihostclaim_test.go
+++ b/internal/console/apihostclaim_test.go
@@ -46,7 +46,7 @@ func TestHostClaimRoutesArePersonOperatorFlagGatedAndInert(t *testing.T) {
if err := json.Unmarshal(body, &out); err != nil {
t.Fatal(err)
}
- if out["error"] != hostClaimPendingRuling {
+ if out["error"] != hostClaimUnavailable {
t.Fatalf("refusal = %q", out["error"])
}
diff --git a/internal/console/apisystemupdate.go b/internal/console/apisystemupdate.go
index 9640e7655..2c9a6fcdc 100644
--- a/internal/console/apisystemupdate.go
+++ b/internal/console/apisystemupdate.go
@@ -459,7 +459,7 @@ func (s *Server) runSystemUpdate(ctx context.Context, id string, job *systemJob,
rel.Artifacts[plat] = release.SystemArtifact{Key: release.SystemArtifactKey(tag, plat), SHA256: sha}
}
}
- p, err := release.PromoteSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), in.Component, in.Channel, rel,
+ p, err := s.promoteHostUpdate(ctx, rel, in.Component, in.Channel,
release.SystemPromoteOptions{Actor: actor, Spread: spread})
if err != nil {
return err
@@ -704,7 +704,7 @@ func (s *Server) handleAPISystemRollback(w http.ResponseWriter, r *http.Request)
jsonError(w, http.StatusBadRequest, "channel must be canary or stable")
return
}
- p, err := release.RollbackSystem(r.Context(), s.cfg.Releases, release.RootSigner(s.cfg.Key), in.Component, in.Channel, addr)
+ p, err := s.rollbackHostUpdate(r.Context(), in.Component, in.Channel, addr)
switch {
case errors.Is(err, release.ErrNoSystemHistory):
jsonError(w, http.StatusConflict, err.Error())
diff --git a/internal/console/hostactivation.go b/internal/console/hostactivation.go
new file mode 100644
index 000000000..521f8a9c2
--- /dev/null
+++ b/internal/console/hostactivation.go
@@ -0,0 +1,227 @@
+package console
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/release"
+ "os"
+ "strings"
+ "time"
+)
+
+func (s *Server) reserveHostUpdate(ctx context.Context, rel *release.SystemRelease, channel, mode string, options release.SystemPromoteOptions, expectedETag ...string) (string, error) {
+ pub := release.RootSigner(s.cfg.Key).RootPublicKey()
+ floor, err := hostfloor.Read(ctx, s.cfg.Releases, pub, s.cfg.Bucket)
+ if errors.Is(err, os.ErrNotExist) {
+ // Legacy mode only when BOTH the floor and its authorization are
+ // absent; any incomplete pair is an unfinished publication, never
+ // the old deployment mode (review round 4).
+ for _, key := range []string{hostfloor.Key, hostfloor.AuthKey} {
+ if _, _, e := s.cfg.Releases.Get(ctx, key); !errors.Is(e, os.ErrNotExist) {
+ return "", hostfloor.ErrFloor
+ }
+ }
+ return "", nil
+ }
+ if err != nil {
+ return "", err
+ }
+ tag := strings.TrimPrefix(rel.Source, "release:")
+ if rel.ReleaseTag != "" {
+ tag = rel.ReleaseTag
+ }
+ if rel.Source == "file" {
+ keys, e := s.cfg.Releases.List(ctx, "control/system/router/releases/")
+ if e != nil {
+ return "", e
+ }
+ tag = ""
+ for _, key := range keys {
+ raw, _, e := s.cfg.Releases.Get(ctx, key)
+ if e != nil {
+ return "", e
+ }
+ identity, e := release.VerifySystemIdentity(pub, raw)
+ if e != nil {
+ return "", e
+ }
+ if identity.Current.Version != rel.Version {
+ continue
+ }
+ matches := true
+ for platform, artifact := range rel.Artifacts {
+ if identity.Current.Artifacts[platform] != artifact {
+ matches = false
+ }
+ }
+ if matches {
+ if tag != "" && tag != identity.Current.ReleaseTag {
+ return "", hostfloor.ErrFloor
+ }
+ tag = identity.Current.ReleaseTag
+ }
+ }
+ }
+ approved, err := release.GetSystemIdentity(ctx, s.cfg.Releases, pub, tag)
+ if err != nil {
+ return "", err
+ }
+ if approved.Current.Version != rel.Version {
+ return "", hostfloor.ErrFloor
+ }
+ for platform, artifact := range rel.Artifacts {
+ if approved.Current.Artifacts[platform] != artifact {
+ return "", hostfloor.ErrFloor
+ }
+ }
+ rel.ReleaseTag = approved.Current.ReleaseTag
+ rel.MovesProtocol = approved.Current.MovesProtocol
+ rel.ConsoleArtifacts = approved.Current.ConsoleArtifacts
+ d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: pub, Bucket: s.cfg.Bucket}
+ o, err := d.Read(ctx)
+ if err != nil {
+ return "", err
+ }
+ requestRelease := *rel
+ requestRelease.ReleasedAt = time.Time{}
+ request, err := json.Marshal(struct {
+ Channel string
+ Mode string
+ Release release.SystemRelease
+ Spread time.Duration
+ }{channel, mode, requestRelease, options.Spread})
+ if err != nil {
+ return "", err
+ }
+ digest := sha256.Sum256(request)
+ requestID := hex.EncodeToString(digest[:])
+ for id, existing := range o.Roster.Activations {
+ if existing.Publication == nil || existing.Publication.Channel != channel {
+ continue
+ }
+ if existing.RequestSHA256 != requestID || existing.FloorSHA256 != floor.Digest {
+ return "", hostactors.ErrRoster
+ }
+ if _, err := existing.Publication.Verify(pub); err != nil {
+ return "", err
+ }
+ return id, nil
+ }
+ var nonce [16]byte
+ if _, err = rand.Read(nonce[:]); err != nil {
+ return "", err
+ }
+ id := hex.EncodeToString(nonce[:])
+ var publication release.SystemPublication
+ if mode == "rollback" {
+ publication, err = release.PrepareSystemRollback(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), "router", channel, options.Actor, id, expectedETag...)
+ } else {
+ publication, err = release.PrepareSystemPromotion(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), "router", channel, *rel, options, id)
+ }
+ if err != nil {
+ return "", err
+ }
+ h := hostactors.Reservation{Publication: &publication, RequestSHA256: requestID, UpdateMode: mode, ID: id, FloorSHA256: floor.Digest, Actors: map[string]string{}, Releases: map[string]hostactors.Report{}, Hashes: map[string][]string{}}
+ e := hostactors.ExactEvidence{PublicKey: d.PublicKey, Store: s.cfg.Releases, Boxes: d.PrivateBoxReport}
+ for ref, b := range o.Roster.Boxes {
+ if b.State != "active" {
+ continue
+ }
+ report, err := d.PrivateBoxReport(ctx, ref, b)
+ if err != nil {
+ return "", err
+ }
+ if err = hostactors.VerifyReport(ctx, e, pub, floor, report, b.Enrollment, b.Cloud.InstanceID, time.Now()); err != nil {
+ return "", err
+ }
+ if report.SystemChannel == "" {
+ return "", hostfloor.ErrFloor
+ }
+ if report.SystemChannel != channel {
+ continue
+ }
+ target, ok := rel.Artifacts[report.Platform]
+ if !ok {
+ return "", hostfloor.ErrFloor
+ }
+ identityKey, _ := release.IdentityKey(rel.ReleaseTag)
+ identity, _, err := s.cfg.Releases.Get(ctx, identityKey)
+ if err != nil {
+ return "", err
+ }
+ if err = hostfloor.CheckImage(pub, floor, identity, rel.Version, report.Platform, target.SHA256); err != nil {
+ return "", err
+ }
+ if target.SHA256 == report.BinarySHA256 {
+ continue
+ }
+ key := "box:" + ref
+ h.Actors[key] = b.Enrollment
+ h.Releases[key] = report
+ // This transaction refreshes recovery from the authenticated current
+ // image. A box unable to do that retains the reservation and refuses.
+ h.Hashes[key] = []string{report.BinarySHA256, target.SHA256, report.BinarySHA256}
+ }
+ if len(h.Actors) == 0 {
+ return "", nil
+ }
+ _, err = d.Reserve(ctx, release.RootSigner(s.cfg.Key), o, h, true)
+ return id, err
+}
+
+func (s *Server) promoteHostUpdate(ctx context.Context, rel release.SystemRelease, component, channel string, options release.SystemPromoteOptions) (*release.SystemPointer, error) {
+ id, err := s.reserveHostUpdate(ctx, &rel, channel, "promote", options)
+ if err != nil {
+ return nil, err
+ }
+ if id == "" {
+ return release.PromoteSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), component, channel, rel, options)
+ }
+ d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: release.RootSigner(s.cfg.Key).RootPublicKey(), Bucket: s.cfg.Bucket}
+ return d.PublishActivation(ctx, id)
+}
+
+func (s *Server) rollbackHostUpdate(ctx context.Context, component, channel, actor string) (*release.SystemPointer, error) {
+ d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: release.RootSigner(s.cfg.Key).RootPublicKey(), Bucket: s.cfg.Bucket}
+ floor, err := hostfloor.Read(ctx, s.cfg.Releases, d.PublicKey, s.cfg.Bucket)
+ if err == nil {
+ roster, err := d.Read(ctx)
+ if err != nil {
+ return nil, err
+ }
+ // A successful publication with a lost response must not pop history again.
+ for id, h := range roster.Roster.Activations {
+ if h.Publication == nil || h.Publication.Channel != channel {
+ continue
+ }
+ if h.UpdateMode != "rollback" || h.FloorSHA256 != floor.Digest {
+ return nil, hostactors.ErrRoster
+ }
+ return d.PublishActivation(ctx, id)
+ }
+ } else if !errors.Is(err, os.ErrNotExist) {
+ return nil, err
+ }
+ current, etag, err := release.GetSystemPointer(ctx, s.cfg.Releases, d.PublicKey, component, channel)
+ if err != nil {
+ return nil, err
+ }
+ if len(current.History) == 0 {
+ return nil, release.ErrNoSystemHistory
+ }
+ target := current.History[0]
+ id, err := s.reserveHostUpdate(ctx, &target, channel, "rollback", release.SystemPromoteOptions{Actor: actor}, etag)
+ if err != nil {
+ return nil, err
+ }
+ if id == "" {
+ return release.RollbackSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), component, channel, actor, etag)
+ }
+ return d.PublishActivation(ctx, id)
+}
diff --git a/internal/console/hostactivation_test.go b/internal/console/hostactivation_test.go
new file mode 100644
index 000000000..c25e54aa7
--- /dev/null
+++ b/internal/console/hostactivation_test.go
@@ -0,0 +1,207 @@
+package console
+
+import (
+ "bytes"
+ "context"
+ "crypto/ed25519"
+ "encoding/json"
+ "errors"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+type activationFaultStore struct {
+ release.Store
+ fail bool
+ commit bool
+ hide bool
+}
+
+func (s *activationFaultStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) {
+ if key != release.SystemPointerKey("router", "stable") || !s.fail {
+ return s.Store.Put(ctx, key, raw, etag)
+ }
+ s.fail = false
+ if s.commit {
+ if _, err := s.Store.Put(ctx, key, raw, etag); err != nil {
+ return "", err
+ }
+ }
+ return "", errors.New("publication response lost")
+}
+func (s *activationFaultStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ if s.hide && key == release.SystemPointerKey("router", "stable") && !s.fail {
+ s.hide = false
+ return nil, "", errors.New("publication read unavailable")
+ }
+ return s.Store.Get(ctx, key)
+}
+
+func activationFixture(t *testing.T) (*Server, hostactors.Directory, release.SystemRelease) {
+ t.Helper()
+ ctx := t.Context()
+ pub, key, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ store := release.NewDirStore(t.TempDir())
+ s := &Server{cfg: Config{Releases: store, Key: key, Bucket: "bucket"}}
+ sg := release.RootSigner(key)
+ old := release.SystemRelease{Version: "v1.0.0", Source: "release:v1.0.0", ReleaseTag: "v1.0.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("v1.0.0", "linux-amd64"), SHA256: strings.Repeat("a", 64)}}}
+ next := release.SystemRelease{Version: "v1.1.0", Source: "release:v1.1.0", ReleaseTag: "v1.1.0", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("v1.1.0", "linux-amd64"), SHA256: strings.Repeat("b", 64)}}}
+ for _, rel := range []release.SystemRelease{next, old} {
+ if _, err := release.PromoteSystem(ctx, store, sg, "router", "stable", rel, release.SystemPromoteOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ raw, _, err := store.Get(ctx, release.SystemPointerKey("router", "stable"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := release.PublishSystemIdentity(ctx, store, pub, raw); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := hostfloor.Publish(ctx, store, sg, "bucket", hostfloor.Floor{Min: 1, MinRelease: "v1.0.0"}); err != nil {
+ t.Fatal(err)
+ }
+ floor, err := hostfloor.Read(ctx, store, pub, "bucket")
+ if err != nil {
+ t.Fatal(err)
+ }
+ enrollment := strings.Repeat("e", 32)
+ roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Publication: "published", FloorSHA256: floor.Digest, Boxes: map[string]hostactors.Box{"box": {BoxID: "box", Namespace: "platform", Label: "box", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-box"}, Enrollment: enrollment, EvidenceSHA256: strings.Repeat("f", 64), State: "active"}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
+ raw, err := release.SignRootEnvelope(sg, roster)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := store.Put(ctx, hostactors.Key, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ box := status.Box{GeneratedAt: time.Now(), Metrics: &status.BoxMetrics{Enrollment: enrollment, InstanceID: "i-box", Release: old.Version, ReleaseTag: old.ReleaseTag, Platform: "linux-amd64", BinarySHA256: old.Artifacts["linux-amd64"].SHA256, MovesProtocol: 1, BootID: "boot", FloorSHA256: floor.Digest, FloorGuardReady: true, HostClaimEnabled: true, SystemChannel: "stable"}}
+ raw, err = json.Marshal(box)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := store.Put(ctx, "status/boxes/box.json", raw, ""); err != nil {
+ t.Fatal(err)
+ }
+ return s, hostactors.Directory{Store: store, PublicKey: pub, Bucket: "bucket"}, next
+}
+
+// Legacy promotion needs both floor objects absent; either half alone is
+// an unfinished publication (review round 4).
+func TestHostUpdateReservationRefusesIncompleteFloorPair(t *testing.T) {
+ for _, drop := range []string{hostfloor.Key, hostfloor.AuthKey} {
+ t.Run(drop, func(t *testing.T) {
+ s, _, target := activationFixture(t)
+ if err := s.cfg.Releases.Delete(t.Context(), drop); err != nil {
+ t.Fatal(err)
+ }
+ if id, err := s.reserveHostUpdate(t.Context(), &target, "stable", "promote", release.SystemPromoteOptions{}); !errors.Is(err, hostfloor.ErrFloor) || id != "" {
+ t.Fatalf("incomplete pair without %s = %q, %v; want ErrFloor", drop, id, err)
+ }
+ })
+ }
+}
+
+func TestHostUpdateReservationRetriesPublication(t *testing.T) {
+ for _, fault := range []string{"crash", "failed", "committed", "unreadable"} {
+ t.Run(fault, func(t *testing.T) {
+ s, d, target := activationFixture(t)
+ id, err := s.reserveHostUpdate(t.Context(), &target, "stable", "promote", release.SystemPromoteOptions{})
+ if err != nil || id == "" {
+ t.Fatal(id, err)
+ }
+ first, err := d.Read(t.Context())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fault != "crash" {
+ broken := &activationFaultStore{Store: d.Store, fail: true, commit: fault != "failed", hide: fault == "unreadable"}
+ s.cfg.Releases = broken
+ _, err = s.promoteHostUpdate(t.Context(), target, "router", "stable", release.SystemPromoteOptions{})
+ if (err == nil) != (fault == "committed") {
+ t.Fatalf("first publication: %v", err)
+ }
+ }
+ // A new console process has no in-memory job or reservation state.
+ s = &Server{cfg: Config{Releases: d.Store, Key: s.cfg.Key, Bucket: "bucket"}}
+ p, err := s.promoteHostUpdate(t.Context(), target, "router", "stable", release.SystemPromoteOptions{})
+ if err != nil || p.ActivationID != id || p.Current.Version != target.Version {
+ t.Fatal(p, err)
+ }
+ after, err := d.Read(t.Context())
+ if err != nil || len(after.Roster.Activations) != 1 {
+ t.Fatal(after, err)
+ }
+ if !bytes.Equal(first.Roster.Activations[id].Publication.After, after.Roster.Activations[id].Publication.After) {
+ t.Fatal("retry replaced publication identity")
+ }
+ })
+ }
+}
+
+func TestHostRollbackResumesWithoutPoppingAgain(t *testing.T) {
+ s, d, _ := activationFixture(t)
+ broken := &activationFaultStore{Store: d.Store, fail: true}
+ s.cfg.Releases = broken
+ if _, err := s.rollbackHostUpdate(t.Context(), "router", "stable", "operator"); err == nil {
+ t.Fatal("missing injected failure")
+ }
+ before, err := d.Read(t.Context())
+ if err != nil || len(before.Roster.Activations) != 1 {
+ t.Fatal(before, err)
+ }
+ var id string
+ for id = range before.Roster.Activations {
+ }
+ s.cfg.Releases = d.Store
+ first, err := s.rollbackHostUpdate(t.Context(), "router", "stable", "operator")
+ if err != nil || first.ActivationID != id || first.Current.Version != "v1.1.0" {
+ t.Fatal(first, err)
+ }
+ second, err := s.rollbackHostUpdate(t.Context(), "router", "stable", "operator")
+ if err != nil || second.ActivationID != id || len(second.History) != len(first.History) {
+ t.Fatal(second, err)
+ }
+}
+
+func TestHostUpdateCancellationBlocksDelayedPublication(t *testing.T) {
+ s, d, target := activationFixture(t)
+ id, err := s.reserveHostUpdate(t.Context(), &target, "stable", "promote", release.SystemPromoteOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ before, err := d.Read(t.Context())
+ if err != nil {
+ t.Fatal(err)
+ }
+ stale := *before.Roster.Activations[id].Publication
+ if err := d.CancelActivation(t.Context(), release.RootSigner(s.cfg.Key), id); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := release.PublishSystemPublication(t.Context(), d.Store, d.PublicKey, stale); !errors.Is(err, release.ErrCASConflict) {
+ t.Fatalf("delayed publication: %v", err)
+ }
+ after, err := d.Read(t.Context())
+ if err != nil || len(after.Roster.Activations) != 0 {
+ t.Fatal(after, err)
+ }
+ p, err := s.promoteHostUpdate(t.Context(), target, "router", "stable", release.SystemPromoteOptions{})
+ if err != nil || p.ActivationID == id {
+ t.Fatal(p, err)
+ }
+ if err := d.CancelActivation(t.Context(), release.RootSigner(s.cfg.Key), p.ActivationID); !errors.Is(err, release.ErrCASConflict) {
+ t.Fatalf("cancelled published activation: %v", err)
+ }
+ after, err = d.Read(t.Context())
+ if err != nil || len(after.Roster.Activations) != 1 {
+ t.Fatal(after, err)
+ }
+}
diff --git a/internal/console/hostreports.go b/internal/console/hostreports.go
new file mode 100644
index 000000000..188d9a1bf
--- /dev/null
+++ b/internal/console/hostreports.go
@@ -0,0 +1,83 @@
+package console
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/version"
+ "log"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "time"
+)
+
+func (s *Server) localHostReport(ctx context.Context) (hostactors.Report, error) {
+ id := s.cfg.HostConsoleID
+ if id == "" {
+ return hostactors.Report{}, hostactors.ErrRoster
+ }
+ d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: s.cfg.ReleasePub, Bucket: s.cfg.Bucket}
+ roster, err := d.Read(ctx)
+ if err != nil {
+ return hostactors.Report{}, err
+ }
+ c, ok := roster.Roster.Consoles[id]
+ if !ok {
+ return hostactors.Report{}, hostactors.ErrRoster
+ }
+ if c.Compute != nil {
+ local, err := hostfloor.ReadEnrollment("/etc/carlos/host-actor.json")
+ box, exists := roster.Roster.Boxes[c.Compute.Box]
+ if err != nil || !exists || local.Box != c.Compute.Box || local.Enrollment != c.Compute.Enrollment || box.Enrollment != local.Enrollment || local.InstanceID != box.Cloud.InstanceID {
+ return hostactors.Report{}, hostactors.ErrRoster
+ }
+ }
+ disk := hostfloor.Disk{Dir: filepath.Join(release.OptDir(), "system", "guard"), Bucket: s.cfg.Bucket, PublicKey: s.cfg.ReleasePub}
+ running, err := disk.Running(version.Build, runtime.GOOS+"-"+runtime.GOARCH, "console")
+ if err != nil {
+ return hostactors.Report{}, err
+ }
+ return hostactors.Report{Compute: c.Compute, Component: "console", GeneratedAt: time.Now().UTC(), Actor: id, Enrollment: c.Enrollment, InstanceID: id, ProcessID: fmt.Sprintf("%s:%d", running.BootID, os.Getpid()), Release: version.Build, ReleaseTag: running.ReleaseTag, Platform: runtime.GOOS + "-" + runtime.GOARCH, BinarySHA256: running.Hash, MovesProtocol: running.Protocol, BootID: running.BootID, FloorSHA256: running.FloorSHA256, FloorGuardReady: running.GuardReady, HostClaimEnabled: s.cfg.HostClaim}, nil
+}
+func (s *Server) StartHostReports(ctx context.Context) {
+ // Reporting before the flag is enabled is harmless and needed for arming.
+ if s.cfg.HostConsoleID == "" || strings.ContainsAny(s.cfg.HostConsoleID, "/\\") || s.cfg.Releases == nil {
+ return
+ }
+ ticker := time.NewTicker(5 * time.Minute)
+ defer ticker.Stop()
+ for {
+ report, err := s.localHostReport(ctx)
+ if err == nil {
+ key := "status/consoles/" + s.cfg.HostConsoleID + ".json"
+ raw, _ := json.Marshal(report)
+ _, etag, readErr := s.cfg.Releases.Get(ctx, key)
+ if readErr != nil && !os.IsNotExist(readErr) {
+ err = readErr
+ } else {
+ _, err = s.cfg.Releases.Put(ctx, key, raw, etag)
+ }
+ }
+ if err == nil && len(s.cfg.Key) != 0 {
+ d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: s.cfg.ReleasePub, Bucket: s.cfg.Bucket}
+ if o, e := d.Read(ctx); e == nil {
+ for id := range o.Roster.Activations {
+ _ = d.ReleaseActivation(ctx, release.RootSigner(s.cfg.Key), id, time.Now())
+ }
+ }
+ }
+ if err != nil {
+ log.Printf("console: host census report: %v", err)
+ }
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ }
+ }
+}
diff --git a/internal/console/server.go b/internal/console/server.go
index 5eaefcbdd..4e67f81b6 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -456,6 +456,8 @@ const (
// Config is everything the server reads its world through.
type Config struct {
+ HostConsoleID string
+
HostClaim bool
DBVersions lease.VersionedStore
Origin string // the console's public origin — the OAuth client_id
diff --git a/internal/hostresolve/readmit.go b/internal/hostresolve/readmit.go
new file mode 100644
index 000000000..c1b539007
--- /dev/null
+++ b/internal/hostresolve/readmit.go
@@ -0,0 +1,131 @@
+package hostresolve
+
+import (
+ "context"
+ "encoding/json"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/hostquarantine"
+ "reflect"
+ "slices"
+ "strings"
+ "time"
+)
+
+// Readmit keeps the global deny until every declaration, claim and receipt
+// check is complete. It grants no executor or writer session.
+func (s Service) Readmit(ctx context.Context, box, enrollment string, revision uint64, lossIDs []string) error {
+ before, err := s.Directory.Read(ctx)
+ if err != nil {
+ return err
+ }
+ b, ok := before.Roster.Boxes[box]
+ if !ok || b.Enrollment != enrollment || b.State != "quarantined" || b.QuarantineRevision != revision || !reflect.DeepEqual(b.LossIDs, lossIDs) {
+ return ErrResolve
+ }
+ fenced, err := s.Directory.TerminationFence(ctx, before, box, s.Now())
+ if err != nil {
+ return err
+ }
+ if fenced {
+ return ErrResolve
+ }
+ report, err := s.Directory.PrivateBoxReport(ctx, box, b)
+ if err != nil {
+ return err
+ }
+ if report.Enrollment != enrollment || report.InstanceID != b.Cloud.InstanceID || report.BootID == "" || report.GeneratedAt.IsZero() || s.Now().Sub(report.GeneratedAt) > 15*time.Minute || report.GeneratedAt.After(s.Now().Add(5*time.Second)) {
+ return ErrResolve
+ }
+ floor, err := hostfloor.Read(ctx, s.Directory.Store, s.Directory.PublicKey, s.Directory.Bucket)
+ if err != nil {
+ return err
+ }
+ evidence := hostactors.ExactEvidence{PublicKey: s.Directory.PublicKey, Store: s.Directory.Store, Boxes: s.Directory.PrivateBoxReport}
+ if err = hostactors.VerifyReport(ctx, evidence, s.Directory.PublicKey, floor, report, enrollment, b.Cloud.InstanceID, s.Now()); err != nil {
+ return err
+ }
+ if !report.HostClaimEnabled {
+ return ErrResolve
+ }
+ key, err := hostquarantine.RetirementKey(box, enrollment, revision)
+ if err != nil {
+ return err
+ }
+ raw, _, err := s.Directory.Store.Get(ctx, key)
+ if err != nil {
+ return err
+ }
+ if report.RetirementReceiptSHA256 != hostfloor.Digest(raw) {
+ return ErrResolve
+ }
+ var receipt hostquarantine.RetirementCompletion
+ if json.Unmarshal(raw, &receipt) != nil || receipt.Validate() != nil || !receipt.Complete || receipt.Box != box || receipt.Enrollment != enrollment || receipt.InstanceID != b.Cloud.InstanceID || receipt.QuarantineRevision != revision || receipt.BootID != report.BootID || receipt.CompletedAt.After(report.GeneratedAt) {
+ return ErrResolve
+ }
+ for _, id := range lossIDs {
+ loss, err := hostclaim.ReadLossRecord(ctx, s.Directory.Store, s.Directory.PublicKey, id)
+ if err != nil {
+ return err
+ }
+ if (loss.Record.State != "committed" && loss.Record.State != "conflicted") || loss.Record.Box != box || loss.Record.Enrollment != enrollment {
+ return ErrResolve
+ }
+ proven := false
+ for _, entry := range receipt.Entries {
+ if entry.Host == loss.Record.Host && entry.Incarnation == loss.Record.HolderIncarnation && entry.Stamp == loss.Record.HolderStamp {
+ proven = true
+ }
+ }
+ if !proven && (loss.Record.State == "conflicted" || len(loss.Record.ConfirmedConsequences.DischargeLocal) == 0) {
+ return ErrResolve
+ }
+ }
+ // Store.List is a complete listing in both real implementations. Its S3
+ // implementation follows every continuation token; any error refuses.
+ keys, err := s.Directory.Store.List(ctx, "control/hosts/")
+ if err != nil {
+ return err
+ }
+ for _, key := range keys {
+ host, ok := claimHost(key)
+ if !ok {
+ continue
+ }
+ observed, err := hostclaim.Read(ctx, s.Directory.Store, host)
+ if err != nil {
+ return err
+ }
+ if observed.Absent {
+ continue
+ }
+ c := observed.Claim
+ for _, o := range c.Obligations {
+ if o.Kind == "park-local" && o.Box == box {
+ return ErrResolve
+ }
+ }
+ if slices.Contains(c.Lost, box) && c.LostRevisions[box] == revision {
+ if _, err = hostclaim.Update(ctx, s.Directory.Store, observed, hostclaim.Readmit(box, revision)); err != nil {
+ return err
+ }
+ }
+ }
+ // CAS against the original roster prevents a concurrent declaration,
+ // reservation or membership change being overwritten by this readmission.
+ next := hostactors.Clone(before.Roster)
+ b.State = "active"
+ next.Boxes[box] = b
+ _, err = s.Directory.Replace(ctx, s.Signer, before, next)
+ return err
+}
+
+func claimHost(key string) (string, bool) {
+ s, ok := strings.CutPrefix(key, "control/hosts/")
+ if !ok || strings.Contains(s, "/") {
+ return "", false
+ }
+ host, ok := strings.CutSuffix(s, ".json")
+ return host, ok && host != ""
+}
diff --git a/internal/hostresolve/review3_test.go b/internal/hostresolve/review3_test.go
new file mode 100644
index 000000000..15e9513da
--- /dev/null
+++ b/internal/hostresolve/review3_test.go
@@ -0,0 +1,243 @@
+package hostresolve
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "reflect"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/hostquarantine"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+func TestPreparedLossClaimProgressAllowsRetiredReadmission(t *testing.T) {
+ for _, retired := range []bool{false, true} {
+ t.Run(map[bool]string{false: "missing-retirement", true: "retired"}[retired], func(t *testing.T) {
+ ctx := t.Context()
+ s, store, now, host := resolveFixture(t)
+ in := submitted(t, s, now, host)
+ store.fail = true
+ if _, err := s.Declare(ctx, "operator", host, in); err == nil {
+ t.Fatal("claim failure not reached")
+ }
+ store.fail = false
+ prepared, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if err != nil || prepared.Record.State != "prepared" {
+ t.Fatal(prepared, err)
+ }
+ before, err := hostclaim.Read(ctx, store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // The retirement owner completes the exact local chore while quarantined.
+ chore := before.Claim.Obligations[0]
+ progressed, err := hostclaim.Update(ctx, store, before, hostclaim.CompleteObligation(chore.Op, chore.ID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = s.Declare(ctx, "operator", host, in); !errors.Is(err, hostclaim.ErrConflict) {
+ t.Fatal("stale declaration must refuse", err)
+ }
+ conflicted, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if err != nil || conflicted.Record.State != "conflicted" {
+ t.Fatal("stale loss not settled", conflicted, err)
+ }
+ want := prepared.Record
+ want.State = "conflicted"
+ if !reflect.DeepEqual(want, conflicted.Record) {
+ t.Fatal("accepted history changed")
+ }
+ if _, err = s.Declare(ctx, "operator", host, in); !errors.Is(err, hostclaim.ErrConflict) {
+ t.Fatal("conflicted retry changed outcome", err)
+ }
+ unchanged, err := hostclaim.Read(ctx, store, host)
+ if err != nil || unchanged.ETag != progressed.ETag {
+ t.Fatal("stale consequences applied", err)
+ }
+ prepareReadmission(t, s, *now, conflicted.Record)
+ if !retired {
+ key, _ := hostquarantine.RetirementKey("dead", conflicted.Record.Enrollment, conflicted.Record.QuarantineRevision)
+ raw, etag, err := store.Get(ctx, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var receipt hostquarantine.RetirementCompletion
+ if err = json.Unmarshal(raw, &receipt); err != nil {
+ t.Fatal(err)
+ }
+ receipt.Entries = []hostquarantine.RetirementEntry{}
+ if err := receipt.Validate(); err != nil {
+ t.Fatal(err)
+ }
+ raw, _ = json.Marshal(receipt)
+ if _, err = store.Put(ctx, key, raw, etag); err != nil {
+ t.Fatal(err)
+ }
+ digest := hostfloor.Digest(raw)
+ raw, etag, err = store.Get(ctx, "status/boxes/dead.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var report status.Box
+ if err = json.Unmarshal(raw, &report); err != nil {
+ t.Fatal(err)
+ }
+ report.Metrics.RetirementReceiptSHA256 = digest
+ raw, _ = json.Marshal(report)
+ if _, err = store.Put(ctx, "status/boxes/dead.json", raw, etag); err != nil {
+ t.Fatal(err)
+ }
+ }
+ roster, err := s.Directory.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b := roster.Roster.Boxes["dead"]
+ err = s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs)
+ if (err == nil) != retired {
+ t.Fatal("readmission must require retirement of conflicted loss", err)
+ }
+ after, err := s.Directory.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ state := "quarantined"
+ if retired {
+ state = "active"
+ }
+ if after.Roster.Boxes["dead"].State != state || !reflect.DeepEqual(after.Roster.Boxes["dead"].LossIDs, b.LossIDs) {
+ t.Fatal("state or loss history changed", after)
+ }
+ retained, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if err != nil || !reflect.DeepEqual(retained.Record, conflicted.Record) {
+ t.Fatal("readmission erased history", err)
+ }
+ })
+ }
+}
+
+type fenceReadmitStore struct {
+ release.Store
+ unreadable bool
+ writes int
+}
+
+func (s *fenceReadmitStore) Get(ctx context.Context, key string) ([]byte, string, error) {
+ if s.unreadable && strings.HasPrefix(key, "control/box-fences/") {
+ return nil, "", errors.New("fence unavailable")
+ }
+ return s.Store.Get(ctx, key)
+}
+func (s *fenceReadmitStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) {
+ s.writes++
+ return s.Store.Put(ctx, key, raw, etag)
+}
+func TestReadmitRefusesTerminationAuthorityBeforeMutation(t *testing.T) {
+ for _, fault := range []string{"terminated", "malformed", "wrong-writer", "unreadable"} {
+ t.Run(fault, func(t *testing.T) {
+ ctx := t.Context()
+ s, store, now, host := resolveFixture(t)
+ in := submitted(t, s, now, host)
+ if _, err := s.Declare(ctx, "operator", host, in); err != nil {
+ t.Fatal(err)
+ }
+ loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if err != nil {
+ t.Fatal(err)
+ }
+ prepareReadmission(t, s, *now, loss.Record)
+ roster, err := s.Directory.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b := roster.Roster.Boxes["dead"]
+ fence := map[string]any{"v": 1, "box": "dead", "enrollment": b.Enrollment, "cloud": b.Cloud, "state": "terminated", "observed_at": *now, "via": "describe", "request_id": "request", "written_by": map[string]string{"box": "dead", "enrollment": b.Enrollment}}
+ if fault == "wrong-writer" {
+ fence["written_by"] = map[string]string{"box": "unknown", "enrollment": b.Enrollment}
+ }
+ raw, _ := json.Marshal(fence)
+ if fault == "malformed" {
+ raw = []byte("{")
+ }
+ if fault != "unreadable" {
+ if err := release.CreateExact(ctx, store, "control/box-fences/"+b.Enrollment+".json", raw); err != nil {
+ t.Fatal(err)
+ }
+ }
+ before, err := hostclaim.Read(ctx, store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ wrapped := &fenceReadmitStore{Store: store, unreadable: fault == "unreadable"}
+ s.Directory.Store = wrapped
+ if err := s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs); err == nil {
+ t.Fatal("fence authority ignored")
+ }
+ if wrapped.writes != 0 {
+ t.Fatal("mutation before fence refusal", wrapped.writes)
+ }
+ after, err := hostclaim.Read(ctx, store, host)
+ if err != nil || after.ETag != before.ETag {
+ t.Fatal("per-host quarantine changed", err)
+ }
+ unchanged, err := s.Directory.Read(ctx)
+ if err != nil || unchanged.ETag != roster.ETag {
+ t.Fatal("global quarantine changed", err)
+ }
+ if unchanged.Roster.Boxes["dead"].State != "quarantined" || len(after.Claim.LossRefs) != 1 {
+ t.Fatal("quarantine or history removed")
+ }
+ })
+ }
+}
+
+type lossBookkeepingStore struct {
+ release.Store
+ fail bool
+}
+
+func (s *lossBookkeepingStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) {
+ if s.fail && etag != "" && strings.HasPrefix(key, "control/host-loss/") {
+ return "", errors.New("loss bookkeeping unavailable")
+ }
+ return s.Store.Put(ctx, key, raw, etag)
+}
+func TestPreparedLossCommitReferenceSurvivesClaimProgress(t *testing.T) {
+ ctx := t.Context()
+ s, store, now, host := resolveFixture(t)
+ in := submitted(t, s, now, host)
+ wrapped := &lossBookkeepingStore{Store: store, fail: true}
+ s.Directory.Store = wrapped
+ if _, err := s.Declare(ctx, "operator", host, in); err == nil {
+ t.Fatal("bookkeeping failure not reached")
+ }
+ loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if err != nil || loss.Record.State != "prepared" {
+ t.Fatal(loss, err)
+ }
+ committed, err := hostclaim.Read(ctx, store, host)
+ if err != nil || len(committed.Claim.LossRefs) != 1 || committed.Claim.LossRefs[0] != in.Confirmation {
+ t.Fatal("claim not committed", err)
+ }
+ progressed, err := hostclaim.Update(ctx, store, committed, func(*hostclaim.Claim) error { return nil })
+ if err != nil {
+ t.Fatal(err)
+ }
+ wrapped.fail = false
+ if _, err := s.Declare(ctx, "operator", host, in); err != nil {
+ t.Fatal("successful CAS treated as conflict", err)
+ }
+ loss, err = hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if err != nil || loss.Record.State != "committed" {
+ t.Fatal(loss, err)
+ }
+ after, err := hostclaim.Read(ctx, store, host)
+ if err != nil || after.ETag != progressed.ETag {
+ t.Fatal("commit recovery changed claim", err)
+ }
+}
diff --git a/internal/hostresolve/service.go b/internal/hostresolve/service.go
new file mode 100644
index 000000000..204a7f9b3
--- /dev/null
+++ b/internal/hostresolve/service.go
@@ -0,0 +1,318 @@
+package hostresolve
+
+import (
+ "context"
+ "crypto/rand"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfence"
+ "github.com/carlosframework/platform/internal/release"
+ "math"
+ "os"
+ "reflect"
+ "slices"
+ "time"
+)
+
+var ErrResolve = errors.New("hostresolve: evidence or operation does not authorize resolution")
+
+type Service struct {
+ Directory hostactors.Directory
+ Signer release.Signer
+ Now func() time.Time
+}
+type Inspection struct {
+ Challenge json.RawMessage `json:"challenge"`
+ Details hostfence.Challenge `json:"details"`
+}
+
+func nonce() (string, error) {
+ var b [16]byte
+ _, err := rand.Read(b[:])
+ return hex.EncodeToString(b[:]), err
+}
+func (s Service) record(ctx context.Context, c *hostclaim.Claim) (hostfence.RecordObservation, error) {
+ _, etag, err := s.Directory.Store.Get(ctx, c.Holder.Key)
+ if errors.Is(err, os.ErrNotExist) {
+ return hostfence.RecordObservation{Key: c.Holder.Key, CommitState: "absent"}, nil
+ }
+ if err != nil {
+ return hostfence.RecordObservation{}, err
+ }
+ return hostfence.RecordObservation{Key: c.Holder.Key, ETag: etag, CommitState: "present"}, nil
+}
+
+// effects implements only the owners present on the Plan 8 baseline. Moves'
+// target-loss policy must come from the moves owner, never from an HTTP flag.
+func effects(c *hostclaim.Claim, box string) (hostfence.Consequences, hostclaim.LossConsequences, error) {
+ var shown hostfence.Consequences
+ var actual hostclaim.LossConsequences
+ if c == nil || c.Holder == nil {
+ return shown, actual, ErrResolve
+ }
+ actual.RecoveryIntent = json.RawMessage(`{"kind":"continue-parking"}`)
+ switch c.Op.Kind {
+ case hostclaim.KindServing:
+ shown.Transition = "destroyed-local-only"
+ actual.RecoveryIntent = json.RawMessage(`{"kind":"destroyed-local-only"}`)
+ for _, o := range c.Obligations {
+ if o.Kind == "park-local" && o.Box == box {
+ ref := hostclaim.ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement}
+ shown.DischargeLocal = append(shown.DischargeLocal, ref)
+ actual.DischargeLocal = append(actual.DischargeLocal, ref)
+ key := o.Op + "/" + o.ID + "/" + o.Retirement
+ shown.DestroyedStorageAssertion = append(shown.DestroyedStorageAssertion, key)
+ actual.DestroyedStorage = append(actual.DestroyedStorage, key)
+ }
+ }
+ if len(actual.DischargeLocal) == 0 {
+ return shown, actual, ErrResolve
+ }
+ case hostclaim.KindDeleting, hostclaim.KindParking:
+ if c.Op.ID == "" || (!slices.Contains(c.Op.Participants, box) && c.Op.Exec != box) {
+ return shown, actual, ErrResolve
+ }
+ shown.Transition = "continue-parking"
+ shown.PossibleWriteLoss = true
+ shown.PossibleTimerLoss = true
+ actual.RevokeExecution = c.Op.Exec == box
+ actual.RemoveParticipation = slices.Contains(c.Op.Participants, box)
+ actual.WritesPotentiallyLost = true
+ actual.TimersPotentiallyLost = true
+ for _, o := range c.Obligations {
+ if (o.Kind == "park-replica" || o.Kind == "park-generation") && o.Owner == "box:"+box {
+ actual.ReassignRegional = append(actual.ReassignRegional, hostclaim.ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement})
+ shown.Chores = append(shown.Chores, o.Op+"/"+o.ID+"/"+o.Retirement)
+ }
+ }
+ default:
+ return shown, actual, ErrResolve
+ }
+ return shown, actual, nil
+}
+func (s Service) Inspect(ctx context.Context, actor, host, name string) (Inspection, error) {
+ ref, b, err := s.Directory.Resolve(ctx, name)
+ if err != nil {
+ return Inspection{}, err
+ }
+ if b.State == "retired" {
+ return Inspection{}, ErrResolve
+ }
+ o, err := hostclaim.Read(ctx, s.Directory.Store, host)
+ if err != nil {
+ return Inspection{}, err
+ }
+ if o.Absent || o.Claim.Holder == nil {
+ return Inspection{}, ErrResolve
+ }
+ shown, _, err := effects(o.Claim, ref)
+ if err != nil {
+ return Inspection{}, err
+ }
+ record, err := s.record(ctx, o.Claim)
+ if err != nil {
+ return Inspection{}, err
+ }
+ id, err := nonce()
+ if err != nil {
+ return Inspection{}, err
+ }
+ c := o.Claim
+ challenge, err := hostfence.PrepareChallenge(s.Now(), hostfence.Challenge{ID: id, Actor: actor, Host: host, HolderIncarnation: c.Holder.Incarnation, HolderStamp: c.Holder.Stamp, HolderSession: c.Holder.Session, ClaimGen: c.Gen, ClaimETag: o.ETag, Op: hostfence.Operation{ID: c.Op.ID, Kind: c.Op.Kind, Phase: c.Op.Phase}, Record: record, Box: ref, Enrollment: b.Enrollment, Cloud: hostfence.CloudBinding{Provider: b.Cloud.Provider, Account: b.Cloud.Account, Region: b.Cloud.Region, InstanceID: b.Cloud.InstanceID}, Consequences: shown})
+ if err != nil {
+ return Inspection{}, err
+ }
+ wire, err := hostfence.SignChallenge(s.Signer, challenge)
+ return Inspection{wire, challenge}, err
+}
+
+type owner struct{}
+
+func (owner) ValidateDeclareLost(c hostclaim.Claim, d hostclaim.LostDeclaration) error {
+ _, want, err := effects(&c, d.Box)
+ if err != nil {
+ return err
+ }
+ if !reflect.DeepEqual(want, d.Consequences) {
+ return ErrResolve
+ }
+ return nil
+}
+func (s Service) Declare(ctx context.Context, actor, host string, in hostfence.Submission) (hostclaim.Observed, error) {
+ now := s.Now().UTC()
+ var c hostfence.Challenge
+ err := release.VerifyRootEnvelope(s.Directory.PublicKey, in.Challenge, &c)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ if c.Actor != actor || c.Host != host || in.Confirmation != c.ID || !in.Attestation.MaintainComputeFence || !in.Attestation.ColdQuarantineStartOnly || !reflect.DeepEqual(in.Attestation.Disks, c.Consequences.DestroyedStorageAssertion) {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ // A retry uses the durable accepted declaration even after its claim CAS.
+ // Inspect a new challenge only when no accepted declaration exists.
+ prior, readErr := hostclaim.ReadLossRecord(ctx, s.Directory.Store, s.Directory.PublicKey, c.ID)
+ if readErr == nil {
+ if prior.Record.Actor != actor || prior.Record.Host != host || prior.Record.Enrollment != c.Enrollment {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ return s.finish(ctx, prior)
+ }
+ if !errors.Is(readErr, hostclaim.ErrLossRecordAbsent) {
+ return hostclaim.Observed{}, readErr
+ }
+ // Acceptance survives a crash before either quarantine or loss materialization.
+ acceptance, readErr := s.readAcceptance(ctx, c.ID)
+ if readErr == nil {
+ if acceptance.Loss.Actor != actor || acceptance.Loss.Host != host || acceptance.Loss.Enrollment != c.Enrollment || acceptance.Loss.Box != c.Box || acceptance.Loss.ExpectedClaimETag != c.ClaimETag {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ return s.accept(ctx, acceptance)
+ }
+ if !errors.Is(readErr, os.ErrNotExist) {
+ return hostclaim.Observed{}, readErr
+ }
+ if _, err = hostfence.VerifyChallenge(s.Directory.PublicKey, in.Challenge, now); err != nil {
+ return hostclaim.Observed{}, err
+ }
+ if err = hostfence.ValidateObservation(c, in.Observation, now); err != nil {
+ return hostclaim.Observed{}, err
+ }
+ roster, err := s.Directory.Read(ctx)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ b, ok := roster.Roster.Boxes[c.Box]
+ if !ok || b.Enrollment != c.Enrollment || b.State == "retired" || b.Cloud != (hostactors.Cloud{Provider: c.Cloud.Provider, Account: c.Cloud.Account, Region: c.Cloud.Region, InstanceID: c.Cloud.InstanceID}) {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ observed, err := hostclaim.Read(ctx, s.Directory.Store, host)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ if observed.Absent || observed.ETag != c.ClaimETag || observed.Claim.Gen != c.ClaimGen {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ record, err := s.record(ctx, observed.Claim)
+ if err != nil || record != c.Record {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ shown, actual, err := effects(observed.Claim, c.Box)
+ if err != nil || !reflect.DeepEqual(shown, c.Consequences) {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ if b.QuarantineRevision == math.MaxUint64 || slices.Contains(b.LossIDs, c.ID) {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ loss := hostclaim.LossRecord{V: 1, ID: c.ID, State: "prepared", Actor: actor, AcceptedAt: now, Host: host, HolderIncarnation: c.HolderIncarnation, HolderStamp: c.HolderStamp, HolderSession: c.HolderSession, ExpectedClaimGen: c.ClaimGen, ExpectedClaimETag: c.ClaimETag, OpID: c.Op.ID, OpKind: c.Op.Kind, OpPhase: c.Op.Phase, Box: c.Box, Enrollment: c.Enrollment, QuarantineRevision: b.QuarantineRevision + 1, FencedAt: now, ConfirmedConsequences: actual, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}}
+ loss.CloudObservation = hostclaim.LossCloudObservation{ObservedAt: in.Observation.ObservedAt, Account: in.Observation.Account, Region: in.Observation.Region, InstanceID: in.Observation.InstanceID, State: in.Observation.State, RequestID: in.Observation.RequestID}
+ loss.OperatorAttestation = hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, DestroyedStorage: in.Attestation.Disks}
+ acceptance = lossAcceptance{V: 1, Bucket: s.Directory.Bucket, RosterETag: roster.ETag, Loss: loss}
+ wire, err := release.SignRootEnvelope(s.Signer, acceptance)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ key, err := acceptanceKey(c.ID)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ if err := release.CreateExact(ctx, s.Directory.Store, key, wire); err != nil {
+ return hostclaim.Observed{}, err
+ }
+ return s.accept(ctx, acceptance)
+}
+
+type lossAcceptance struct {
+ V int `json:"v"`
+ Bucket string `json:"bucket"`
+ RosterETag string `json:"roster_etag"`
+ Loss hostclaim.LossRecord `json:"loss"`
+}
+
+func acceptanceKey(id string) (string, error) {
+ if _, err := hostclaim.LossRecordKey(id); err != nil {
+ return "", err
+ }
+ return "control/host-loss-acceptances/" + id + ".json", nil
+}
+func (s Service) readAcceptance(ctx context.Context, id string) (lossAcceptance, error) {
+ key, err := acceptanceKey(id)
+ if err != nil {
+ return lossAcceptance{}, err
+ }
+ raw, _, err := s.Directory.Store.Get(ctx, key)
+ if err != nil {
+ return lossAcceptance{}, err
+ }
+ var a lossAcceptance
+ if release.VerifyRootEnvelope(s.Directory.PublicKey, raw, &a) != nil || a.V != 1 || a.Bucket != s.Directory.Bucket || a.RosterETag == "" || a.Loss.ID != id || a.Loss.State != "prepared" || a.Loss.Validate() != nil {
+ return lossAcceptance{}, ErrResolve
+ }
+ return a, nil
+}
+func (s Service) accept(ctx context.Context, a lossAcceptance) (hostclaim.Observed, error) {
+ roster, err := s.Directory.Read(ctx)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ loss := a.Loss
+ b, ok := roster.Roster.Boxes[loss.Box]
+ if !ok || b.Enrollment != loss.Enrollment || b.State == "retired" || b.Cloud.Account != loss.CloudObservation.Account || b.Cloud.Region != loss.CloudObservation.Region || b.Cloud.InstanceID != loss.CloudObservation.InstanceID {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ if !slices.Contains(b.LossIDs, loss.ID) {
+ // Retry unrelated roster changes with a fresh CAS. A declaration for
+ // this same box defeats the accepted revision and needs new inspection.
+ if b.QuarantineRevision == math.MaxUint64 || b.QuarantineRevision+1 != loss.QuarantineRevision {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ roster, err = s.Directory.Quarantine(ctx, s.Signer, roster, loss.Box, loss.Enrollment, loss.ID)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ b = roster.Roster.Boxes[loss.Box]
+ }
+ if b.State != "quarantined" || b.QuarantineRevision < loss.QuarantineRevision {
+ return hostclaim.Observed{}, ErrResolve
+ }
+ // The global deny remains in force through every subsequent failure.
+ prepared, err := hostclaim.CreateLossRecord(ctx, s.Directory.Store, s.Signer, loss)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ return s.finish(ctx, prepared)
+}
+
+func (s Service) finish(ctx context.Context, loss hostclaim.ObservedLossRecord) (hostclaim.Observed, error) {
+ observed, err := hostclaim.Read(ctx, s.Directory.Store, loss.Record.Host)
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ if loss.Record.State == "conflicted" {
+ return hostclaim.Observed{}, hostclaim.ErrConflict
+ }
+ // A retained loss reference proves an earlier successful CAS even if
+ // bookkeeping failed. Otherwise a changed ETag fences every stale retry.
+ if loss.Record.State == "prepared" && (observed.Absent || !slices.Contains(observed.Claim.LossRefs, loss.Record.ID)) && (observed.Absent || observed.ETag != loss.Record.ExpectedClaimETag || observed.Claim.Gen != loss.Record.ExpectedClaimGen) {
+ r := loss.Record
+ r.State = "conflicted"
+ if _, err := hostclaim.ReplaceLossRecord(ctx, s.Directory.Store, s.Signer, loss, r); err != nil {
+ return hostclaim.Observed{}, err
+ }
+ return hostclaim.Observed{}, hostclaim.ErrConflict
+ }
+ next, err := hostclaim.DeclareLost(ctx, s.Directory.Store, s.Directory.PublicKey, observed, loss.Record.ID, owner{})
+ if err != nil {
+ return hostclaim.Observed{}, err
+ }
+ if loss.Record.State == "committed" {
+ return next, nil
+ }
+ r := loss.Record
+ r.State = "committed"
+ r.ResultClaimGen = next.Claim.Gen
+ _, err = hostclaim.ReplaceLossRecord(ctx, s.Directory.Store, s.Signer, loss, r)
+ return next, err
+}
diff --git a/internal/hostresolve/service_test.go b/internal/hostresolve/service_test.go
new file mode 100644
index 000000000..85904db05
--- /dev/null
+++ b/internal/hostresolve/service_test.go
@@ -0,0 +1,252 @@
+package hostresolve
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfence"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/hostquarantine"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+ "strings"
+ "testing"
+ "time"
+)
+
+type failClaimStore struct {
+ release.Store
+ fail bool
+ crash string
+}
+
+func (s *failClaimStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) {
+ if s.fail && strings.HasPrefix(key, "control/hosts/") {
+ return "", errors.New("claim write unavailable")
+ }
+ if s.crash == "acceptance" && strings.HasPrefix(key, "control/host-loss-acceptances/") || s.crash == "before-quarantine" && key == hostactors.Key || s.crash == "loss-create" && strings.HasPrefix(key, "control/host-loss/") {
+ return "", errors.New("crash before durable write")
+ }
+ next, err := s.Store.Put(ctx, key, raw, etag)
+ if err == nil && s.crash == "after-quarantine" && key == hostactors.Key {
+ return "", errors.New("crash after durable quarantine")
+ }
+ return next, err
+}
+func resolveFixture(t *testing.T) (Service, *failClaimStore, *time.Time, string) {
+ t.Helper()
+ ctx := context.Background()
+ pub, key, _ := ed25519.GenerateKey(rand.Reader)
+ store := &failClaimStore{Store: release.NewDirStore(t.TempDir())}
+ signer := release.RootSigner(key)
+ now := time.Now().UTC()
+ d := hostactors.Directory{Store: store, PublicKey: pub, Bucket: "bucket"}
+ if err := d.Initialize(ctx, signer); err != nil {
+ t.Fatal(err)
+ }
+ o, err := d.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ r := hostactors.Clone(o.Roster)
+ r.Boxes["dead"] = hostactors.Box{Enrollment: strings.Repeat("a", 32), Namespace: "platform", BoxID: "dead", Label: "dead-label", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-dead"}, EvidenceSHA256: strings.Repeat("b", 64), State: "active"}
+ if _, err = d.Replace(ctx, signer, o, r); err != nil {
+ t.Fatal(err)
+ }
+ host := "shop.bab.oncarlos.com"
+ observed, _ := hostclaim.Read(ctx, store, host)
+ n := 0
+ _, err = hostclaim.Update(ctx, store, observed, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + host + ".json", Account: "bab", App: "jam", Host: host, Region: "ie", RecordRegion: "ie"}, "live", nil, []string{"dead"}, "discovery", func() string { n++; return fmt.Sprintf("id-%d", n) }))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return Service{Directory: d, Signer: signer, Now: func() time.Time { return now }}, store, &now, host
+}
+func submitted(t *testing.T, s Service, now *time.Time, host string) hostfence.Submission {
+ t.Helper()
+ in, err := s.Inspect(context.Background(), "operator", host, "dead-label")
+ if err != nil {
+ t.Fatal(err)
+ }
+ *now = now.Add(2 * time.Second)
+ return hostfence.Submission{Challenge: in.Challenge, Confirmation: in.Details.ID, Observation: hostfence.CloudObservation{ObservedAt: in.Details.IssuedAt.Add(time.Second), Account: "123", Region: "eu-west-1", InstanceID: "i-dead", State: "stopped", RequestID: "request"}, Attestation: hostfence.OperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, Disks: in.Details.Consequences.DestroyedStorageAssertion}}
+}
+func TestLostWiringPreservesServingAuthorityAndRetryEvidence(t *testing.T) {
+ ctx := context.Background()
+ s, store, now, host := resolveFixture(t)
+ in := submitted(t, s, now, host)
+ before, _ := hostclaim.Read(ctx, store, host)
+ after, err := s.Declare(ctx, "operator", host, in)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if after.Claim.Op.Exec != before.Claim.Op.Exec || after.Claim.Holder.Stamp != before.Claim.Holder.Stamp || len(after.Claim.Obligations) != 0 {
+ t.Fatal(after.Claim)
+ }
+ loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if err != nil || loss.Record.State != "committed" {
+ t.Fatal(loss, err)
+ }
+ first := loss.Record.FencedAt
+ *now = now.Add(time.Hour)
+ if _, err = s.Declare(ctx, "operator", host, in); err != nil {
+ t.Fatal("accepted retry expired", err)
+ }
+ loss, _ = hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if !loss.Record.FencedAt.Equal(first) {
+ t.Fatal("retry moved fence time")
+ }
+ if _, err = s.Declare(ctx, "other-person", host, in); err == nil {
+ t.Fatal("actor changed")
+ }
+}
+func TestFailedClaimWriteLeavesGlobalQuarantine(t *testing.T) {
+ s, store, now, host := resolveFixture(t)
+ in := submitted(t, s, now, host)
+ store.fail = true
+ if _, err := s.Declare(context.Background(), "operator", host, in); err == nil {
+ t.Fatal("expected write error")
+ }
+ actor, err := s.Directory.Lookup(context.Background(), "dead")
+ if err != nil || actor.State != "quarantined" {
+ t.Fatal(actor, err)
+ }
+ store.fail = false
+ if _, err := s.Declare(context.Background(), "operator", host, in); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestDeclareCrashAndChallengeExpiryAllowRetryAndReadmission(t *testing.T) {
+ for _, crash := range []string{"acceptance", "before-quarantine", "after-quarantine", "loss-create"} {
+ t.Run(crash, func(t *testing.T) {
+ ctx := t.Context()
+ s, store, now, host := resolveFixture(t)
+ in := submitted(t, s, now, host)
+ acceptedAt := *now
+ before, err := hostclaim.Read(ctx, store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ store.crash = crash
+ if _, err := s.Declare(ctx, "operator", host, in); err == nil {
+ t.Fatal("crash not reached")
+ }
+ unchanged, err := hostclaim.Read(ctx, store, host)
+ if err != nil || unchanged.ETag != before.ETag {
+ t.Fatal("claim changed before quarantine completion", err)
+ }
+ roster, err := s.Directory.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b := roster.Roster.Boxes["dead"]
+ if crash == "acceptance" || crash == "before-quarantine" {
+ if b.State != "active" || len(b.LossIDs) != 0 {
+ t.Fatal("quarantined before acceptance", b)
+ }
+ } else if b.State != "quarantined" || len(b.LossIDs) != 1 || b.LossIDs[0] != in.Confirmation {
+ t.Fatal("deny not retained", b)
+ }
+ if _, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation); !errors.Is(err, hostclaim.ErrLossRecordAbsent) {
+ t.Fatal("loss unexpectedly materialized", err)
+ }
+ if crash != "acceptance" {
+ if _, err := s.readAcceptance(ctx, in.Confirmation); err != nil {
+ t.Fatal("acceptance missing", err)
+ }
+ }
+ store.crash = ""
+ if crash == "before-quarantine" {
+ // An unrelated roster CAS must not strand accepted evidence.
+ if _, err := s.Directory.Replace(ctx, s.Signer, roster, hostactors.Clone(roster.Roster)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ *now = now.Add(time.Hour)
+ if _, err := hostfence.VerifyChallenge(s.Directory.PublicKey, in.Challenge, *now); err == nil {
+ t.Fatal("challenge did not expire")
+ }
+ if crash == "acceptance" {
+ if _, err := s.Declare(ctx, "operator", host, in); err == nil {
+ t.Fatal("unaccepted expired submission allowed")
+ }
+ return
+ }
+ if _, err := s.Declare(ctx, "operator", host, in); err != nil {
+ t.Fatal("accepted retry expired", err)
+ }
+ loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if err != nil || loss.Record.State != "committed" || !loss.Record.AcceptedAt.Equal(acceptedAt) || !loss.Record.FencedAt.Equal(acceptedAt) {
+ t.Fatal("acceptance changed", loss, err)
+ }
+ prepareReadmission(t, s, *now, loss.Record)
+ roster, err = s.Directory.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ b = roster.Roster.Boxes["dead"]
+ if err := s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs); err != nil {
+ t.Fatal("orphan blocked readmission", err)
+ }
+ actor, err := s.Directory.Lookup(ctx, "dead")
+ if err != nil || actor.State != "active" {
+ t.Fatal("readmission failed", actor, err)
+ }
+ })
+ }
+}
+
+func prepareReadmission(t *testing.T, s Service, now time.Time, loss hostclaim.LossRecord) {
+ t.Helper()
+ ctx := t.Context()
+ store := s.Directory.Store
+ floor := hostfloor.Floor{Min: 1, MinRelease: "v1.2.3"}
+ if err := hostfloor.Publish(ctx, store, s.Signer, s.Directory.Bucket, floor); err != nil {
+ t.Fatal(err)
+ }
+ f, err := hostfloor.Read(ctx, store, s.Directory.PublicKey, s.Directory.Bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ hash := strings.Repeat("d", 64)
+ rel := release.SystemRelease{Version: "build123", ReleaseTag: floor.MinRelease, MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{"linux-amd64": {Key: release.SystemArtifactKey("build123", "linux-amd64"), SHA256: hash}}}
+ if _, err := release.PromoteSystem(ctx, store, s.Signer, "router", "stable", rel, release.SystemPromoteOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ wire, _, err := store.Get(ctx, release.SystemPointerKey("router", "stable"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := release.PublishSystemIdentity(ctx, store, s.Directory.PublicKey, wire); err != nil {
+ t.Fatal(err)
+ }
+ receipt := hostquarantine.RetirementCompletion{V: 1, Box: loss.Box, Enrollment: loss.Enrollment, InstanceID: loss.CloudObservation.InstanceID, QuarantineRevision: loss.QuarantineRevision, BootID: "new-boot", CompletedAt: now, Complete: true, InventorySHA256: strings.Repeat("e", 64), Entries: []hostquarantine.RetirementEntry{{Host: loss.Host, Incarnation: loss.HolderIncarnation, Stamp: loss.HolderStamp, RetirementID: "retirement", JournalSHA256: strings.Repeat("f", 64)}}}
+ if err := receipt.Validate(); err != nil {
+ t.Fatal(err)
+ }
+ raw, err := json.Marshal(receipt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ key, err := hostquarantine.RetirementKey(loss.Box, loss.Enrollment, loss.QuarantineRevision)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := release.CreateExact(ctx, store, key, raw); err != nil {
+ t.Fatal(err)
+ }
+ report := status.Box{GeneratedAt: now, Label: "dead-label", Metrics: &status.BoxMetrics{Enrollment: loss.Enrollment, InstanceID: loss.CloudObservation.InstanceID, Release: rel.Version, ReleaseTag: rel.ReleaseTag, Platform: "linux-amd64", BinarySHA256: hash, MovesProtocol: 1, BootID: receipt.BootID, FloorSHA256: f.Digest, FloorGuardReady: true, HostClaimEnabled: true, RetirementReceiptSHA256: hostfloor.Digest(raw)}}
+ raw, err = json.Marshal(report)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := release.CreateExact(ctx, store, "status/boxes/"+loss.Box+".json", raw); err != nil {
+ t.Fatal(err)
+ }
+}