| 1 | //go:build unix |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "sync" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/hostbox" |
| 18 | "github.com/carlosframework/platform/internal/hostclaim" |
| 19 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 20 | "github.com/carlosframework/platform/internal/hostguard" |
| 21 | "github.com/carlosframework/platform/internal/release" |
| 22 | ) |
| 23 | |
| 24 | // guardClaims is the guard's claim reader, set by the test. |
| 25 | type guardClaims struct { |
| 26 | mu sync.Mutex |
| 27 | obs hostclaim.Observed |
| 28 | } |
| 29 | |
| 30 | func (g *guardClaims) set(o hostclaim.Observed) { g.mu.Lock(); g.obs = o; g.mu.Unlock() } |
| 31 | func (g *guardClaims) Read(context.Context, string) (hostclaim.Observed, error) { |
| 32 | g.mu.Lock() |
| 33 | defer g.mu.Unlock() |
| 34 | return g.obs, nil |
| 35 | } |
| 36 | |
| 37 | // failFirstClaimRead fails the first Get of the claim key only: C10's |
| 38 | // re-read (Astra round 4). |
| 39 | type failFirstClaimRead struct { |
| 40 | release.Store |
| 41 | key string |
| 42 | failed bool |
| 43 | mu sync.Mutex |
| 44 | } |
| 45 | |
| 46 | var errClaimReadDown = errors.New("control store unreachable") |
| 47 | |
| 48 | func (s *failFirstClaimRead) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 49 | s.mu.Lock() |
| 50 | first := key == s.key && !s.failed |
| 51 | if first { |
| 52 | s.failed = true |
| 53 | } |
| 54 | s.mu.Unlock() |
| 55 | if first { |
| 56 | return nil, "", errClaimReadDown |
| 57 | } |
| 58 | return s.Store.Get(ctx, key) |
| 59 | } |
| 60 | |
| 61 | // Astra round 4 (rev 4.2, ruling 70): after a restart between the serving |
| 62 | // CAS and the participation discharge, a failure of only C10's claim |
| 63 | // re-read never falls through to ordinary cleanup. Cleanup fails with that |
| 64 | // error, nothing is stopped, the completed probe's fence is not replaced; |
| 65 | // on retry the participation is discharged by the binding and the probe's |
| 66 | // receipt, and the first serving wake's cold handover (N6's prior, the |
| 67 | // probe's receipt) is accepted by the guard. |
| 68 | func TestRelocationRestartDischargeSurvivesAFailedClaimRead(t *testing.T) { |
| 69 | ctx := t.Context() |
| 70 | const host, box = "shop.example", "dst" |
| 71 | created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| 72 | inc, err := hostclaim.Incarnation("acct", "app", host, created) |
| 73 | if err != nil { |
| 74 | t.Fatal(err) |
| 75 | } |
| 76 | holder := func(session, stamp uint64, gen string) *hostclaim.Holder { |
| 77 | return &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created, |
| 78 | Location: "instances/" + host, Stamp: stamp, Session: session, Generation: gen, WriterBox: box, |
| 79 | Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}} |
| 80 | } |
| 81 | probing := hostclaim.Observed{Host: host, ETag: "e1", Claim: &hostclaim.Claim{V: 2, Gen: 30, Host: host, Holder: holder(12, 9, "P1"), |
| 82 | Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Exec: box, Participants: []string{box}}}} |
| 83 | reader := &guardClaims{obs: probing} |
| 84 | dataDir := t.TempDir() |
| 85 | if err := os.Mkdir(filepath.Join(dataDir, "hostguard"), 0o770); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | guard := hostguard.Guard{DataDir: dataDir, LockDir: t.TempDir(), Claims: reader} |
| 89 | if err := guard.Activate(ctx, host, box, nil); err != nil { |
| 90 | t.Fatal(err) |
| 91 | } |
| 92 | // The probe's writer-session stop, completed with root's receipt |
| 93 | // (ruling 49), while the op was still relocating/probing. |
| 94 | prior := hostguard.FenceTarget{FenceID: "fence-1", Region: "ie", Host: host, Incarnation: inc, Stamp: 9, Session: 12, Op: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, |
| 95 | ObligationID: hostguard.WriterStopObligation(12), Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", ClaimETag: "e1", DirectoryRevision: "1", RosterRevision: 1} |
| 96 | if _, err := guard.BeginFence(ctx, prior); err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | at := time.Date(2026, 10, 11, 9, 0, 0, 0, time.UTC) |
| 100 | receipt := mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r1", Kind: StopReceiptStopped, Region: "ie", ClaimETag: "e1", ObligationID: prior.ObligationID, FenceID: "fence-1", Host: host, Incarnation: inc, |
| 101 | InstallationStamp: 9, WriterSession: 12, OperationID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", DirectoryRevision: testReceiptNumeral(prior), Binding: "1", EvidenceAt: at}) |
| 102 | if err := guard.CompleteFence(ctx, prior, receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { |
| 103 | t.Fatal(err) |
| 104 | } |
| 105 | sum := sha256.Sum256(receipt) |
| 106 | done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: box, Session: 12, Receipt: "fence-1", ReceiptSHA256: hex.EncodeToString(sum[:])}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| 107 | serving := &hostclaim.Claim{V: 2, Gen: 31, Host: host, Holder: holder(12, 9, "P1"), |
| 108 | Op: hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}} |
| 109 | raw, err := json.Marshal(serving) |
| 110 | if err != nil { |
| 111 | t.Fatal(err) |
| 112 | } |
| 113 | key, _ := hostclaim.Key(host) |
| 114 | dir := release.NewDirStore(t.TempDir()) |
| 115 | if _, err := dir.Put(ctx, key, raw, ""); err != nil { |
| 116 | t.Fatal(err) |
| 117 | } |
| 118 | if obs, err := hostclaim.Read(ctx, dir, host); err != nil || obs.Claim.Op.Kind != hostclaim.KindServing { |
| 119 | t.Fatalf("the serving claim does not read back: %v", err) |
| 120 | } |
| 121 | reader.set(hostclaim.Observed{Host: host, ETag: "e2", Claim: serving}) |
| 122 | store := &failFirstClaimRead{Store: dir, key: key} |
| 123 | stops := 0 |
| 124 | a := &ClaimAssembly{Claims: store, Guard: guard, Wake: &ClaimWakeAssembly{}, Launches: &hostbox.LaunchJournal{}, |
| 125 | Reconcile: &hostbox.ReconcileAdapter{Census: &hostbox.EmptyInstallationCensus{}}, |
| 126 | collectStop: func(context.Context, StopRequest, StopDrainer) (StopResult, error) { |
| 127 | stops++ |
| 128 | return StopResult{}, errors.New("a serving-shaped stop") |
| 129 | }} |
| 130 | p := &productionClaimLocal{legacyParticipation: true, assembly: a, slots: map[string]*claimParticipation{}} |
| 131 | // The restart: clearUntracked hands cleanup the current serving |
| 132 | // observation and the persisted .hostexec binding. |
| 133 | binding := execlease.Binding{Box: box, Enrollment: "enr", Host: host, Incarnation: inc, Op: "rel-1", BootID: "boot-0"} |
| 134 | current := hostclaim.Observed{Host: host, ETag: "e2", Claim: serving} |
| 135 | |
| 136 | if err := p.cleanup(ctx, current, binding); !errors.Is(err, errClaimReadDown) { |
| 137 | t.Fatalf("a failed C10 read fell through: %v", err) |
| 138 | } |
| 139 | if stops != 0 { |
| 140 | t.Fatal("ordinary cleanup asked a stop while the claim read was failing") |
| 141 | } |
| 142 | if got, err := guard.CompletedWriterStops(ctx, host, hostguard.Captured{Incarnation: inc, Stamp: 9}, "rel-1", box); err != nil || len(got) != 1 || got[0].Target != prior { |
| 143 | t.Fatalf("the probe's receipt changed: %+v %v", got, err) |
| 144 | } |
| 145 | |
| 146 | // The retry: storage recovered. |
| 147 | if err := p.cleanup(ctx, current, binding); err != nil { |
| 148 | t.Fatalf("the retry did not discharge: %v", err) |
| 149 | } |
| 150 | if stops != 0 { |
| 151 | t.Fatal("the discharge asked a stop") |
| 152 | } |
| 153 | // The first serving wake: N6's Before verifies the receipt, stopping |
| 154 | // nothing; after its cold admission (session and stamp 40), the |
| 155 | // guard's handover barrier over the probe's receipt is accepted, |
| 156 | // which a replacement fence would have refused (ErrFenceActive). |
| 157 | h := relocationCompletedHandover{Guard: guard, Box: box} |
| 158 | if err := h.Before(ctx, current); err != nil { |
| 159 | t.Fatalf("N6 before admission: %v", err) |
| 160 | } |
| 161 | r, err := h.receipt(ctx, current) |
| 162 | if err != nil || r.Target != prior { |
| 163 | t.Fatalf("N6's prior is not the probe's receipt: %+v %v", r.Target, err) |
| 164 | } |
| 165 | admitted := *serving |
| 166 | admitted.Holder = holder(40, 40, "g40") |
| 167 | admitted.Holder.Admissions = append(admitted.Holder.Admissions, hostclaim.AdmissionRecord{Session: 40, Box: box, Generation: "g40", Mode: "cold", Region: "ie"}) |
| 168 | reader.set(hostclaim.Observed{Host: host, ETag: "e3", Claim: &admitted}) |
| 169 | barrier := prior |
| 170 | barrier.FenceID, barrier.Stamp, barrier.Session, barrier.Op, barrier.Phase, barrier.ObligationID, barrier.ClaimETag = "handover-1", 40, 40, "", "", "", "e3" |
| 171 | if err := guard.BeginColdHandover(ctx, prior, barrier, func(*hostclaim.Claim) error { return nil }); err != nil { |
| 172 | t.Fatalf("the serving handover from the probe's receipt: %v", err) |
| 173 | } |
| 174 | } |
| 175 | |
amadan