amadan

carlos / platform Public

host, hostresolve, hostactors, console: console guard starts serve, readmission clears every covered revision, pre-admission console reports (Plan 8b review)

Co-Authored-By: Codex <noreply@openai.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
codex pushed by claude 38088a79405810ffe94089cf153d0e00071e0b75 parent 7649d9f
9 files changed, +451 −9
  • internal/console/hostreports.go +12 −3
  • internal/console/hostreports_test.go +145 −0
  • internal/host/guardarm.go +1 −1
  • internal/hostactors/console_invitation.go +58 −0
  • internal/hostactors/console_invitation_test.go +39 −0
  • internal/hostfloor/report.go +5 −1
  • internal/hostfloor/report_test.go +21 −0
  • internal/hostresolve/readmit.go +20 −4
  • internal/hostresolve/readmit_test.go +150 −0

CI passed — run details

diff --git a/internal/console/hostreports.go b/internal/console/hostreports.go
index 188d9a1bf..b56aaf512 100644
--- a/internal/console/hostreports.go
+++ b/internal/console/hostreports.go
@@ -17,6 +17,13 @@ import (
)
func (s *Server) localHostReport(ctx context.Context) (hostactors.Report, error) {
+ disk := hostfloor.Disk{Dir: filepath.Join(release.OptDir(), "system", "guard"), Bucket: s.cfg.Bucket, PublicKey: s.cfg.ReleasePub}
+ return s.hostReport(ctx, func() (hostfloor.Running, error) {
+ return disk.Running(version.Build, runtime.GOOS+"-"+runtime.GOARCH, "console")
+ })
+}
+
+func (s *Server) hostReport(ctx context.Context, runningImage func() (hostfloor.Running, error)) (hostactors.Report, error) {
id := s.cfg.HostConsoleID
if id == "" {
return hostactors.Report{}, hostactors.ErrRoster
@@ -28,7 +35,10 @@ func (s *Server) localHostReport(ctx context.Context) (hostactors.Report, error)
}
c, ok := roster.Roster.Consoles[id]
if !ok {
- return hostactors.Report{}, hostactors.ErrRoster
+ c, err = d.InvitedConsole(ctx, roster, id)
+ if err != nil {
+ return hostactors.Report{}, err
+ }
}
if c.Compute != nil {
local, err := hostfloor.ReadEnrollment("/etc/carlos/host-actor.json")
@@ -37,8 +47,7 @@ func (s *Server) localHostReport(ctx context.Context) (hostactors.Report, error)
return hostactors.Report{}, hostactors.ErrRoster
}
}
- disk := hostfloor.Disk{Dir: filepath.Join(release.OptDir(), "system", "guard"), Bucket: s.cfg.Bucket, PublicKey: s.cfg.ReleasePub}
- running, err := disk.Running(version.Build, runtime.GOOS+"-"+runtime.GOARCH, "console")
+ running, err := runningImage()
if err != nil {
return hostactors.Report{}, err
}
diff --git a/internal/console/hostreports_test.go b/internal/console/hostreports_test.go
new file mode 100644
index 000000000..80bc9131a
--- /dev/null
+++ b/internal/console/hostreports_test.go
@@ -0,0 +1,145 @@
+package console
+
+import (
+ "crypto/ed25519"
+ "encoding/json"
+ "errors"
+ "reflect"
+ "runtime"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostactors"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/version"
+)
+
+func TestHostConsoleAdmissionAfterFloorPublication(t *testing.T) {
+ build := version.Build
+ version.Build = "build123"
+ t.Cleanup(func() { version.Build = build })
+ for _, fault := range []string{"none", "missing", "unsigned", "wrong-signer", "wrong-bucket", "wrong-id", "wrong-enrollment", "wrong-compute", "unready", "stale", "report-enrollment", "guard-error"} {
+ t.Run(fault, func(t *testing.T) {
+ ctx := t.Context()
+ s, d, _ := activationFixture(t)
+ s.cfg.HostConsoleID, s.cfg.ReleasePub, s.cfg.HostClaim = "console-2", d.PublicKey, true
+ signer := release.RootSigner(s.cfg.Key)
+ before, err := d.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if before.Roster.Publication != "published" {
+ t.Fatal("fixture has no published floor")
+ }
+ c := hostactors.Console{Enrollment: strings.Repeat("c", 32), InstanceID: s.cfg.HostConsoleID, Namespaces: []string{"platform"}, DeploymentBucket: d.Bucket}
+ invited := c
+ bucket := d.Bucket
+ switch fault {
+ case "wrong-bucket":
+ bucket = "other"
+ case "wrong-id":
+ invited.InstanceID = "other"
+ case "wrong-enrollment":
+ invited.Enrollment = "invalid"
+ case "wrong-compute":
+ invited.Compute = &hostactors.ComputeBinding{Box: "unknown", Enrollment: strings.Repeat("a", 32)}
+ }
+ invitation := hostactors.ConsoleInvitation{V: 1, Bucket: bucket, Console: invited}
+ invitationSigner := signer
+ if fault == "wrong-signer" {
+ _, key, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ invitationSigner = release.RootSigner(key)
+ }
+ wire, err := release.SignRootEnvelope(invitationSigner, invitation)
+ if fault == "unsigned" {
+ wire, err = json.Marshal(invitation)
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fault != "missing" {
+ if err := release.CreateExact(ctx, d.Store, "control/host-console-invitations/"+c.InstanceID+".json", wire); err != nil {
+ t.Fatal(err)
+ }
+ }
+ platform := runtime.GOOS + "-" + runtime.GOARCH
+ hash := strings.Repeat("d", 64)
+ rel := release.SystemRelease{Version: version.Build, ReleaseTag: "v1.2.3", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{platform: {Key: release.SystemArtifactKey(version.Build, platform), SHA256: hash}}, ConsoleArtifacts: map[string]release.SystemArtifact{platform: {Key: "bootstrap/console/build123/carlos-" + platform, SHA256: hash}}}
+ if _, err := release.PromoteSystem(ctx, d.Store, signer, "router", "stable", rel, release.SystemPromoteOptions{}); err != nil {
+ t.Fatal(err)
+ }
+ wire, _, err = d.Store.Get(ctx, release.SystemPointerKey("router", "stable"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := release.PublishSystemIdentity(ctx, d.Store, d.PublicKey, wire); err != nil {
+ t.Fatal(err)
+ }
+ next := hostactors.Clone(before.Roster)
+ next.Consoles[c.InstanceID] = c
+ if _, err := d.Replace(ctx, signer, before, next); err == nil {
+ t.Fatal("invitation admitted console without a report")
+ }
+ called := false
+ report, err := s.hostReport(ctx, func() (hostfloor.Running, error) {
+ called = true
+ if fault == "guard-error" {
+ return hostfloor.Running{}, hostfloor.ErrFloor
+ }
+ return hostfloor.Running{ReleaseTag: rel.ReleaseTag, Hash: hash, Protocol: 1, BootID: "boot", FloorSHA256: before.Roster.FloorSHA256, GuardReady: fault != "unready"}, nil
+ })
+ refused := fault == "missing" || fault == "unsigned" || fault == "wrong-signer" || fault == "wrong-bucket" || fault == "wrong-id" || fault == "wrong-enrollment" || fault == "wrong-compute"
+ if refused || fault == "guard-error" {
+ if err == nil || (refused && called) {
+ t.Fatalf("invalid invitation or guard accepted: called=%v, err=%v", called, err)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatal("invited console cannot report before admission", err)
+ }
+ if report.Actor != c.InstanceID || report.Enrollment != c.Enrollment || report.Component != "console" || report.ProcessID == "" {
+ t.Fatal("incorrect reporting identity", report)
+ }
+ unchanged, err := d.Read(ctx)
+ if err != nil || unchanged.ETag != before.ETag {
+ t.Fatal("reporting changed membership", err)
+ }
+ switch fault {
+ case "stale":
+ report.GeneratedAt = time.Now().Add(-time.Hour)
+ case "report-enrollment":
+ report.Enrollment = strings.Repeat("b", 32)
+ }
+ raw, err := json.Marshal(report)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := release.CreateExact(ctx, d.Store, "status/consoles/"+c.InstanceID+".json", raw); err != nil {
+ t.Fatal(err)
+ }
+ after, err := d.Replace(ctx, signer, before, next)
+ if fault != "none" {
+ if err == nil {
+ t.Fatal("invalid report admitted console")
+ }
+ return
+ }
+ if err != nil || !reflect.DeepEqual(after.Roster.Consoles[c.InstanceID], c) {
+ t.Fatal("admission after reporting failed", err)
+ }
+ if err := d.Store.Delete(ctx, "control/host-console-invitations/"+c.InstanceID+".json"); err != nil {
+ t.Fatal(err)
+ }
+ _, err = s.hostReport(ctx, func() (hostfloor.Running, error) { return hostfloor.Running{}, hostfloor.ErrFloor })
+ if !errors.Is(err, hostfloor.ErrFloor) {
+ t.Fatal("admitted console still requires invitation", err)
+ }
+ })
+ }
+}
diff --git a/internal/host/guardarm.go b/internal/host/guardarm.go
index 84d00f15b..e315d13e7 100644
--- a/internal/host/guardarm.go
+++ b/internal/host/guardarm.go
@@ -156,7 +156,7 @@ func RunSystemGuardArm(args []string) {
log.Fatal(hostfloor.ErrFloor)
}
}
- dropins["console.service"] = "[Service]\nExecStart=\nExecStart=/opt/carlos/system/guard/carlos system guard-exec console " + *consolePath + "\n"
+ dropins["console.service"] = hostfloor.ConsoleGuardDropIn(*consolePath)
}
for unit, body := range dropins {
dir := filepath.Join("/etc/systemd/system", unit+".d")
diff --git a/internal/hostactors/console_invitation.go b/internal/hostactors/console_invitation.go
new file mode 100644
index 000000000..2238fcade
--- /dev/null
+++ b/internal/hostactors/console_invitation.go
@@ -0,0 +1,58 @@
+package hostactors
+
+import (
+ "context"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+type ConsoleInvitation struct {
+ V int `json:"v"`
+ Bucket string `json:"bucket"`
+ Console Console `json:"console"`
+}
+
+func (d Directory) InviteConsole(ctx context.Context, signer release.Signer, c Console) error {
+ o, err := d.Read(ctx)
+ if err != nil {
+ return err
+ }
+ if err := d.validateInvitedConsole(o, c.InstanceID, c); err != nil {
+ return err
+ }
+ wire, err := release.SignRootEnvelope(signer, ConsoleInvitation{V: 1, Bucket: d.Bucket, Console: c})
+ if err != nil {
+ return err
+ }
+ return release.CreateExact(ctx, d.Store, "control/host-console-invitations/"+c.InstanceID+".json", wire)
+}
+
+func (d Directory) InvitedConsole(ctx context.Context, o Observed, id string) (Console, error) {
+ if !segment(id) {
+ return Console{}, ErrRoster
+ }
+ wire, _, err := d.Store.Get(ctx, "control/host-console-invitations/"+id+".json")
+ if err != nil {
+ return Console{}, err
+ }
+ var invitation ConsoleInvitation
+ if release.VerifyRootEnvelope(d.PublicKey, wire, &invitation) != nil || invitation.V != 1 || invitation.Bucket != d.Bucket {
+ return Console{}, ErrRoster
+ }
+ if err := d.validateInvitedConsole(o, id, invitation.Console); err != nil {
+ return Console{}, err
+ }
+ return invitation.Console, nil
+}
+
+func (d Directory) validateInvitedConsole(o Observed, id string, c Console) error {
+ if !segment(id) || c.InstanceID != id {
+ return ErrRoster
+ }
+ if _, exists := o.Roster.Consoles[id]; exists {
+ return ErrRoster
+ }
+ next := Clone(o.Roster)
+ next.Consoles[id] = c
+ return next.Validate(d.Bucket)
+}
diff --git a/internal/hostactors/console_invitation_test.go b/internal/hostactors/console_invitation_test.go
new file mode 100644
index 000000000..f9cb972bd
--- /dev/null
+++ b/internal/hostactors/console_invitation_test.go
@@ -0,0 +1,39 @@
+package hostactors
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestConsoleInvitationDoesNotAdmitOrRebind(t *testing.T) {
+ ctx := t.Context()
+ d, signer, before := fixture(t)
+ c := Console{Enrollment: strings.Repeat("c", 32), InstanceID: "console-2", Namespaces: []string{"platform"}, DeploymentBucket: d.Bucket}
+ for range 2 {
+ if err := d.InviteConsole(ctx, signer, c); err != nil {
+ t.Fatal(err)
+ }
+ }
+ got, err := d.InvitedConsole(ctx, before, c.InstanceID)
+ if err != nil || !reflect.DeepEqual(got, c) {
+ t.Fatal("invitation identity changed", got, err)
+ }
+ changed := c
+ changed.Enrollment = strings.Repeat("d", 32)
+ if err := d.InviteConsole(ctx, signer, changed); err == nil {
+ t.Fatal("rebound invitation")
+ }
+ changed = c
+ changed.InstanceID = "../other"
+ if err := d.InviteConsole(ctx, signer, changed); err == nil {
+ t.Fatal("invalid console ID invited")
+ }
+ if _, err := d.InvitedConsole(ctx, before, "../other"); err == nil {
+ t.Fatal("invalid console ID read")
+ }
+ after, err := d.Read(ctx)
+ if err != nil || after.ETag != before.ETag {
+ t.Fatal("invitation changed roster membership", err)
+ }
+}
diff --git a/internal/hostfloor/report.go b/internal/hostfloor/report.go
index c0f1a319a..3fde2007b 100644
--- a/internal/hostfloor/report.go
+++ b/internal/hostfloor/report.go
@@ -37,7 +37,7 @@ func (d Disk) Running(build, platform string, role ...string) (Running, error) {
if err != nil {
return Running{}, err
}
- want := "[Service]\nExecStart=\nExecStart=/opt/carlos/system/guard/carlos system guard-exec console " + string(path) + "\n"
+ want := ConsoleGuardDropIn(string(path))
if string(dropin) != want {
return Running{}, ErrFloor
}
@@ -97,3 +97,7 @@ func ReadEnrollment(path string) (Enrollment, error) {
}
return e, nil
}
+
+func ConsoleGuardDropIn(path string) string {
+ return "[Service]\nExecStart=\nExecStart=/opt/carlos/system/guard/carlos system guard-exec console " + path + " serve\n"
+}
diff --git a/internal/hostfloor/report_test.go b/internal/hostfloor/report_test.go
new file mode 100644
index 000000000..6e755ddc4
--- /dev/null
+++ b/internal/hostfloor/report_test.go
@@ -0,0 +1,21 @@
+package hostfloor
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestConsoleGuardDropInStartsServe(t *testing.T) {
+ const path = "/opt/carlos/console/current/console"
+ body := ConsoleGuardDropIn(path)
+ prefix := "[Service]\nExecStart=\nExecStart="
+ if !strings.HasPrefix(body, prefix) {
+ t.Fatalf("drop-in does not reset ExecStart: %q", body)
+ }
+ args := strings.Fields(strings.TrimPrefix(body, prefix))
+ want := []string{"/opt/carlos/system/guard/carlos", "system", "guard-exec", "console", path, "serve"}
+ if !reflect.DeepEqual(args, want) {
+ t.Fatalf("guarded console invocation = %q, want %q", args, want)
+ }
+}
diff --git a/internal/hostresolve/readmit.go b/internal/hostresolve/readmit.go
index c1b539007..f029df39c 100644
--- a/internal/hostresolve/readmit.go
+++ b/internal/hostresolve/readmit.go
@@ -64,12 +64,13 @@ func (s Service) Readmit(ctx context.Context, box, enrollment string, revision u
if json.Unmarshal(raw, &receipt) != nil || receipt.Validate() != nil || !receipt.Complete || receipt.Box != box || receipt.Enrollment != enrollment || receipt.InstanceID != b.Cloud.InstanceID || receipt.QuarantineRevision != revision || receipt.BootID != report.BootID || receipt.CompletedAt.After(report.GeneratedAt) {
return ErrResolve
}
+ losses := make(map[string][]hostclaim.LossRecord)
for _, id := range lossIDs {
loss, err := hostclaim.ReadLossRecord(ctx, s.Directory.Store, s.Directory.PublicKey, id)
if err != nil {
return err
}
- if (loss.Record.State != "committed" && loss.Record.State != "conflicted") || loss.Record.Box != box || loss.Record.Enrollment != enrollment {
+ if (loss.Record.State != "committed" && loss.Record.State != "conflicted") || loss.Record.Box != box || loss.Record.Enrollment != enrollment || loss.Record.QuarantineRevision > revision {
return ErrResolve
}
proven := false
@@ -81,6 +82,7 @@ func (s Service) Readmit(ctx context.Context, box, enrollment string, revision u
if !proven && (loss.Record.State == "conflicted" || len(loss.Record.ConfirmedConsequences.DischargeLocal) == 0) {
return ErrResolve
}
+ losses[loss.Record.Host] = append(losses[loss.Record.Host], loss.Record)
}
// Store.List is a complete listing in both real implementations. Its S3
// implementation follows every continuation token; any error refuses.
@@ -88,6 +90,7 @@ func (s Service) Readmit(ctx context.Context, box, enrollment string, revision u
if err != nil {
return err
}
+ var claims []hostclaim.Observed
for _, key := range keys {
host, ok := claimHost(key)
if !ok {
@@ -106,10 +109,23 @@ func (s Service) Readmit(ctx context.Context, box, enrollment string, revision u
return ErrResolve
}
}
- if slices.Contains(c.Lost, box) && c.LostRevisions[box] == revision {
- if _, err = hostclaim.Update(ctx, s.Directory.Store, observed, hostclaim.Readmit(box, revision)); err != nil {
- return err
+ if slices.Contains(c.Lost, box) {
+ covered := false
+ for _, loss := range losses[host] {
+ if loss.State == "committed" && loss.QuarantineRevision == c.LostRevisions[box] && slices.Contains(c.LossRefs, loss.ID) {
+ covered = true
+ break
+ }
}
+ if !covered {
+ return ErrResolve
+ }
+ claims = append(claims, observed)
+ }
+ }
+ for _, observed := range claims {
+ if _, err = hostclaim.Update(ctx, s.Directory.Store, observed, hostclaim.Readmit(box, observed.Claim.LostRevisions[box])); err != nil {
+ return err
}
}
// CAS against the original roster prevents a concurrent declaration,
diff --git a/internal/hostresolve/readmit_test.go b/internal/hostresolve/readmit_test.go
new file mode 100644
index 000000000..8413c13c1
--- /dev/null
+++ b/internal/hostresolve/readmit_test.go
@@ -0,0 +1,150 @@
+package hostresolve
+
+import (
+ "encoding/json"
+ "fmt"
+ "reflect"
+ "slices"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostfloor"
+ "github.com/carlosframework/platform/internal/hostquarantine"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+func TestReadmitClearsEveryCoveredHostRevision(t *testing.T) {
+ for _, fault := range []string{"none", "uncovered-revision", "missing-loss-reference", "future-revision"} {
+ t.Run(fault, func(t *testing.T) {
+ ctx := t.Context()
+ s, store, now, first := resolveFixture(t)
+ hosts := []string{first, "zzz.bab.oncarlos.com"}
+ other, err := hostclaim.Read(ctx, store, hosts[1])
+ if err != nil {
+ t.Fatal(err)
+ }
+ n := 0
+ _, err = hostclaim.Update(ctx, store, other, hostclaim.Adopt(hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + hosts[1] + ".json", Account: "bab", App: "jam", Host: hosts[1], Region: "ie", RecordRegion: "ie"}, "live", nil, []string{"dead"}, "discovery", func() string { n++; return fmt.Sprintf("other-%d", n) }))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var losses []hostclaim.LossRecord
+ for _, host := range hosts {
+ in := submitted(t, s, now, host)
+ if _, err := s.Declare(ctx, "operator", host, in); err != nil {
+ t.Fatal(err)
+ }
+ loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
+ if err != nil {
+ t.Fatal(err)
+ }
+ losses = append(losses, loss.Record)
+ }
+ prepareReadmission(t, s, *now, losses[1])
+ key, err := hostquarantine.RetirementKey("dead", losses[1].Enrollment, 2)
+ if err != nil {
+ t.Fatal(err)
+ }
+ raw, etag, err := store.Get(ctx, key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var receipt hostquarantine.RetirementCompletion
+ if err := json.Unmarshal(raw, &receipt); err != nil {
+ t.Fatal(err)
+ }
+ receipt.Entries = append(receipt.Entries, hostquarantine.RetirementEntry{Host: first, Incarnation: losses[0].HolderIncarnation, Stamp: losses[0].HolderStamp, RetirementID: "earlier-retirement", JournalSHA256: receipt.Entries[0].JournalSHA256})
+ raw, err = json.Marshal(receipt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = store.Put(ctx, key, raw, etag); err != nil {
+ t.Fatal(err)
+ }
+ digest := hostfloor.Digest(raw)
+ raw, etag, err = store.Get(ctx, "status/boxes/dead.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var report status.Box
+ if err := json.Unmarshal(raw, &report); err != nil {
+ t.Fatal(err)
+ }
+ report.Metrics.RetirementReceiptSHA256 = digest
+ raw, err = json.Marshal(report)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = store.Put(ctx, "status/boxes/dead.json", raw, etag); err != nil {
+ t.Fatal(err)
+ }
+ if fault != "none" {
+ other, err = hostclaim.Read(ctx, store, hosts[1])
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, err = hostclaim.Update(ctx, store, other, func(c *hostclaim.Claim) error {
+ switch fault {
+ case "uncovered-revision":
+ c.LostRevisions["dead"] = 1
+ case "missing-loss-reference":
+ c.LossRefs = nil
+ case "future-revision":
+ c.LostRevisions["dead"] = 3
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ before := make([]hostclaim.Observed, len(hosts))
+ for i, host := range hosts {
+ before[i], err = hostclaim.Read(ctx, store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+ roster, err := s.Directory.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ box := roster.Roster.Boxes["dead"]
+ if box.QuarantineRevision != 2 {
+ t.Fatal("expected two retained revisions", box)
+ }
+ err = s.Readmit(ctx, "dead", box.Enrollment, box.QuarantineRevision, box.LossIDs)
+ if (err == nil) != (fault == "none") {
+ t.Fatalf("Readmit = %v, fault = %s", err, fault)
+ }
+ for i, host := range hosts {
+ after, err := hostclaim.Read(ctx, store, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fault != "none" {
+ if after.ETag != before[i].ETag {
+ t.Fatal("invalid inventory changed a claim", host)
+ }
+ continue
+ }
+ if slices.Contains(after.Claim.Lost, "dead") || after.Claim.LostRevisions["dead"] != 0 {
+ t.Errorf("readmission stranded %s at revision %d", host, after.Claim.LostRevisions["dead"])
+ }
+ if !reflect.DeepEqual(after.Claim.LossRefs, before[i].Claim.LossRefs) {
+ t.Fatal("retained loss references changed", host)
+ }
+ }
+ after, err := s.Directory.Read(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fault != "none" && after.ETag != roster.ETag {
+ t.Fatal("invalid inventory cleared global quarantine")
+ }
+ if fault == "none" && after.Roster.Boxes["dead"].State != "active" {
+ t.Fatal("box was not readmitted")
+ }
+ })
+ }
+}