amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostclaim
2
3import (
4 "context"
5 "crypto/sha256"
6 "encoding/hex"
7 "encoding/json"
8 "errors"
9 "reflect"
10 "slices"
11 "strconv"
12 "strings"
13 "testing"
14 "time"
15
16 "github.com/carlosframework/platform/internal/boxid"
17 "github.com/carlosframework/platform/internal/release"
18 "github.com/carlosframework/platform/internal/termfence"
19)
20
21const (
22 relSource = "adi-knitting"
23 relTarget = "bri-knitting"
24 relSpare = "cai-knitting"
25 relSpare2 = "dov-knitting"
26)
27
28var relObserved = time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC)
29
30func relEnrollment(box string) string { return strings.Repeat(string(box[0]), 32) }
31
32// relAuthority accepts any fence record for a box in this region.
33type relAuthority struct{}
34
35func (relAuthority) ValidateFence(_ context.Context, f termfence.Record) (FenceBinding, error) {
36 if f.Enrollment != relEnrollment(f.Box) {
37 return FenceBinding{}, ErrFence
38 }
39 return FenceBinding{Region: "ie", RosterRevision: 9, RosterETag: "roster"}, nil
40}
41
42type relReleases struct{ err error }
43
44func (r relReleases) VerifyRelease(context.Context, Claim, Release) error { return r.err }
45
46func putFence(t *testing.T, s release.Store, box string) FenceRef {
47 t.Helper()
48 f := termfence.Record{V: 1, Box: box, Enrollment: relEnrollment(box), State: "terminated",
49 Cloud: boxid.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + box},
50 ObservedAt: relObserved.Format(time.RFC3339Nano), Via: "describe", RequestID: "req",
51 WrittenBy: termfence.WrittenBy{Box: "worker", Enrollment: strings.Repeat("e", 32)}}
52 raw, err := json.Marshal(f)
53 if err != nil {
54 t.Fatal(err)
55 }
56 sum := sha256.Sum256(raw)
57 ref := FenceRef{Enrollment: f.Enrollment, Key: "control/box-fences/" + f.Enrollment + ".json", SHA256: hex.EncodeToString(sum[:])}
58 if _, err := s.Put(context.Background(), ref.Key, raw, ""); err != nil && !errors.Is(err, release.ErrCASConflict) {
59 t.Fatal(err)
60 }
61 return ref
62}
63
64// relServing is a serving claim on relSource whose source has admitted one
65// cold writer session (generation g1).
66func relServing(t *testing.T) (release.Store, Observed) {
67 t.Helper()
68 ctx := context.Background()
69 s := release.NewDirStore(t.TempDir())
70 o := servingClaim(t, ctx, s)
71 o, err := Update(ctx, s, o, AdmitWriter(Admission{Box: relSource, Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true}))
72 if err != nil {
73 t.Fatal(err)
74 }
75 return s, o
76}
77
78func relStart(reason string, fence *FenceRef) RelocationStart {
79 return RelocationStart{OpID: "rel-1", By: "worker", Now: relObserved, Region: "ie", Reason: reason,
80 SourceBox: relSource, SourceEnrollment: relEnrollment(relSource),
81 TargetBox: relTarget, TargetEnrollment: relEnrollment(relTarget),
82 DirectoryRevision: 9, AdmissionID: "adm-1", Fence: fence}
83}
84
85func step(t *testing.T, s release.Store, o Observed, tr func(*Claim) error) Observed {
86 t.Helper()
87 next, err := Update(context.Background(), s, o, tr)
88 if err != nil {
89 t.Fatal(err)
90 }
91 return next
92}
93
94func ack(box string) func(*Claim) error {
95 return AcknowledgeExecution(box, ExecutionAcknowledgement{Op: "rel-1", Enrollment: relEnrollment(box), BootID: "boot", ControllerID: "ctl", LeaseETag: "lease", Expires: relObserved.Add(time.Hour)})
96}
97
98// relRelease is the app release the test pins name (R9).
99const relRelease = "v1.4.2"
100
101// relPin is a pin selected against o: its key names o's claim generation.
102func relPin(o Observed, maxTXID uint64, seq int64, at time.Time) RecoveryPin {
103 return RecoveryPin{Key: RecoveryPinKey(o.Claim.Host, "rel-1", o.Claim.Gen), SHA256: strings.Repeat("a", 64), Generation: "g1", MaxTXID: maxTXID, FilesManifest: "instances/x/files/g1/manifest.json", FilesSeq: seq, PinnedAt: at, Empty: maxTXID == 0, Release: relRelease}
104}
105
106func relPrepared(gen string, maxTXID uint64, filesSeq int64) PreparedAuthority {
107 a := PreparedAuthority{Generation: gen, SnapshotTXID: maxTXID, FilesSeq: 0, SourceFilesSeq: filesSeq, FilesManifestSHA256: strings.Repeat("f", 64)}
108 if maxTXID > 0 {
109 a.InstallManifest, a.SHA256 = "m", strings.Repeat("c", 64)
110 }
111 return a
112}
113
114// journalN is journalUint for a journal the test's own transitions wrote.
115func journalN(op Op, key string) uint64 {
116 n, err := journalUint(op, key)
117 if err != nil {
118 panic(err)
119 }
120 return n
121}
122
123func localInstall(c *Claim) LocalInstall {
124 return LocalInstall{Box: c.Op.Exec, Enrollment: relEnrollment(c.Op.Exec), InstallStamp: journalN(c.Op, "install_stamp"), FilesManifestVersion: 3, JournalSHA256: strings.Repeat("b", 64)}
125}
126
127// toProbing drives an operation at pinning (with its pin) through
128// installing, installed and the local install receipt into probing.
129func toProbing(t *testing.T, s release.Store, o Observed, prepared string, maxTXID uint64) Observed {
130 t.Helper()
131 target := Target{Box: o.Claim.Op.Intent["target_box"], Enrollment: o.Claim.Op.Intent["target_enrollment"]}
132 o = step(t, s, o, ToInstalling("rel-1", target, prepared))
133 var pin RecoveryPin
134 readJournal(o.Claim.Op, "pin", &pin)
135 o = step(t, s, o, ToInstalled("rel-1", relPrepared(prepared, maxTXID, pin.FilesSeq)))
136 o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
137 return step(t, s, o, ToProbing("rel-1", relObserved))
138}
139
140func TestRelocationDrainEndToEnd(t *testing.T) {
141 ctx := context.Background()
142 s, o := relServing(t)
143 o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{})
144 if err != nil {
145 t.Fatal(err)
146 }
147 if c := o.Claim; c.Op.Exec != relSource || c.Op.Phase != PhaseStarting || !slices.Equal(c.Op.Participants, []string{relSource, relTarget}) || c.Op.Intent["stamp_from"] == "" {
148 t.Fatalf("start: %+v", c.Op)
149 }
150 if _, err := Update(ctx, s, o, ToReleasing("rel-1", relObserved)); !errors.Is(err, ErrRelocation) {
151 t.Fatalf("barrier without acknowledgements: %v", err)
152 }
153 o = step(t, s, o, ack(relSource))
154 o = step(t, s, o, ack(relTarget))
155 o = step(t, s, o, ToReleasing("rel-1", relObserved))
156 if o.Claim.Op.Journal["releasing_since"] == "" {
157 t.Fatal("releasing did not start its deadline")
158 }
159 h := o.Claim.Holder
160 rel := Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "control/stop/r.json", ReceiptSHA256: strings.Repeat("d", 64)}
161 o = step(t, s, o, JournalRelease("rel-1", rel))
162 if _, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{err: errors.New("bad receipt")}, nil); err == nil {
163 t.Fatal("pinned on an unverified receipt")
164 }
165 if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil {
166 t.Fatal(err)
167 }
168 stamp := o.Claim.Holder.Stamp
169 if o.Claim.Op.Exec != "none" || o.Claim.Op.Phase != PhasePinning {
170 t.Fatalf("pinning: %+v", o.Claim.Op)
171 }
172 if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 39, 7, relObserved))); !errors.Is(err, ErrRelocation) {
173 t.Fatalf("pin short of the release: %v", err)
174 }
175 stale := relPin(o, 40, 7, relObserved)
176 stale.Key = RecoveryPinKey(o.Claim.Host, "rel-1", o.Claim.Gen-1)
177 if _, err := Update(ctx, s, o, JournalPin("rel-1", stale)); !errors.Is(err, ErrRelocation) {
178 t.Fatalf("a pin selected against an earlier observation was journaled: %v", err)
179 }
180 o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved)))
181 if o.Claim.Holder.Stamp != stamp {
182 t.Fatal("pinning changed the stamp")
183 }
184 o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1"))
185 for name, a := range map[string]PreparedAuthority{
186 "no files manifest": {Generation: "P1", InstallManifest: "m", SHA256: strings.Repeat("c", 64), SnapshotTXID: 40},
187 "another files seq": {Generation: "P1", InstallManifest: "m", SHA256: strings.Repeat("c", 64), SnapshotTXID: 40, FilesSeq: 6, FilesManifestSHA256: strings.Repeat("f", 64)},
188 "no install manifest": {Generation: "P1", SnapshotTXID: 40, FilesSeq: 7, FilesManifestSHA256: strings.Repeat("f", 64)},
189 "another snapshot txid": {Generation: "P1", InstallManifest: "m", SHA256: strings.Repeat("c", 64), SnapshotTXID: 41, FilesSeq: 7, FilesManifestSHA256: strings.Repeat("f", 64)},
190 } {
191 if _, err := Update(ctx, s, o, ToInstalled("rel-1", a)); !errors.Is(err, ErrRelocation) {
192 t.Fatalf("%s: installed on an incomplete build: %v", name, err)
193 }
194 }
195 o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7)))
196 o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
197 o = step(t, s, o, ToProbing("rel-1", relObserved))
198 install := journalN(o.Claim.Op, "install_stamp")
199 if o.Claim.Op.Exec != relTarget || o.Claim.Holder.Stamp != install || !slices.Contains(o.Claim.Touched.Boxes, relTarget) {
200 t.Fatalf("installation: exec %s stamp %d install %d", o.Claim.Op.Exec, o.Claim.Holder.Stamp, install)
201 }
202 o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
203 Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"}))
204 last := o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1]
205 if last.Coverage != "" || last.WritesLost != nil || o.Claim.Holder.Stamp != install {
206 t.Fatalf("certified admission: %+v stamp %d", last, o.Claim.Holder.Stamp)
207 }
208 o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session}))
209 o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}))
210 o = step(t, s, o, CompleteRelocation("rel-1", counter()))
211 c := o.Claim
212 if c.Op.Kind != KindServing || c.Op.ID != "" || c.Op.Exec != relTarget || !slices.Equal(c.Op.Participants, []string{relTarget}) {
213 t.Fatalf("serving: %+v", c.Op)
214 }
215 if len(c.Obligations) != 1 || c.Obligations[0].Kind != "park-local" || c.Obligations[0].Box != relSource || c.Obligations[0].Op != "rel-1" {
216 t.Fatalf("the source's park-local: %+v", c.Obligations)
217 }
218}
219
220func TestRelocationFailoverCitesTheFenceAndDeclaresTheLoss(t *testing.T) {
221 ctx := context.Background()
222 for _, qualified := range []bool{false, true} {
223 s, o := relServing(t)
224 o = step(t, s, o, func(c *Claim) error { // a chore the dead source already owed
225 c.Obligations = append(c.Obligations, Obligation{ID: "old", Op: "old-op", Kind: "park-local", Box: relSource, Retirement: "r", Owner: "box:" + relSource})
226 return nil
227 })
228 ref := putFence(t, s, relSource)
229 sourceSession := o.Claim.Holder.Session
230 in := relStart(ReasonFailover, &ref)
231 in.Discharge = &LocalDischarge{Qualified: qualified, Records: []string{"control/box-volumes/x/1.json"}}
232 o, err := StartRelocation(ctx, s, o, in, relAuthority{})
233 if err != nil {
234 t.Fatal(err)
235 }
236 c := o.Claim
237 if c.Op.Exec != "none" || !slices.Equal(c.Op.Participants, []string{relTarget}) || !slices.Contains(c.Lost, relSource) || c.Op.Journal["fence."+relSource] == "" {
238 t.Fatalf("failover start: %+v lost %v", c.Op, c.Lost)
239 }
240 if owes := slices.ContainsFunc(c.Obligations, func(o Obligation) bool { return o.Box == relSource }); owes == qualified {
241 t.Fatalf("discharge qualified=%v left chores %+v", qualified, c.Obligations)
242 }
243 if _, err := Update(ctx, s, o, ToReleasing("rel-1", relObserved)); err == nil {
244 t.Fatal("a failover entered releasing")
245 }
246 o = step(t, s, o, ack(relTarget))
247 o = step(t, s, o, PinFailover("rel-1"))
248 if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved.Add(29*time.Second)))); !errors.Is(err, ErrRelocation) {
249 t.Fatalf("pinned inside the 30-second window: %v", err)
250 }
251 o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved.Add(FailoverPinDelay))))
252 o = toProbing(t, s, o, "P1", 40)
253 probe := Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
254 Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"}
255 if _, err := Update(ctx, s, o, AdmitWriter(probe)); !errors.Is(err, ErrWrongState) {
256 t.Fatalf("a failover admission certified its predecessor without a release: %v", err)
257 }
258 probe.PredecessorCertified = false
259 o = step(t, s, o, AdmitWriter(probe))
260 last := o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1]
261 if last.WritesLost == nil || last.WritesLost.Box != relSource || last.WritesLost.Generation != "g1" {
262 t.Fatalf("failover must declare the source's session: %+v", last)
263 }
264 o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session}))
265 o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}))
266 o = step(t, s, o, CompleteRelocation("rel-1", counter()))
267 owes := slices.ContainsFunc(o.Claim.Obligations, func(ob Obligation) bool { return ob.Kind == "park-local" && ob.Box == relSource })
268 if owes == qualified {
269 t.Fatalf("qualified=%v: source chores at serving %+v", qualified, o.Claim.Obligations)
270 }
271 // D4: the loss outlives the operation, with the console still to
272 // materialize it (nothing here stands in for the console).
273 c = o.Claim
274 if len(c.EvidenceRefs) != 1 || len(c.LossRefs) != 1 || c.EvidenceRefs[0].ID != c.LossRefs[0] {
275 t.Fatalf("retained losses after serving: %+v refs %v", c.EvidenceRefs, c.LossRefs)
276 }
277 l := c.EvidenceRefs[0]
278 if l.Box != relSource || l.Fence != ref || l.ObservedAt != relObserved.Format(time.RFC3339Nano) || l.Op != "rel-1" || l.Role != "source" ||
279 l.Session != sourceSession || l.Generation != "g1" || l.Discharge.Qualified != qualified || l.Timers != TimersUnknown || l.Reported {
280 t.Fatalf("retained loss: %+v", l)
281 }
282 done, ok, _ := ReadCompletedRelocation(c)
283 if len(c.Op.Journal) != 1 || !ok || done.Op != "rel-1" || done.Stop.Session != c.Holder.Session || done.Phase != PhaseRelocatingProbing || !slices.Contains(c.Lost, relSource) {
284 t.Fatalf("serving kept the journal, or lost the box: %+v %v", c.Op, c.Lost)
285 }
286 }
287}
288
289func TestRelocationSourceLossBecomesFailover(t *testing.T) {
290 ctx := context.Background()
291 for _, tc := range []struct {
292 name string
293 prepare func(*testing.T, release.Store, Observed) Observed
294 wantPhase string
295 wantExec string
296 wantParts []string
297 }{
298 {"starting before acknowledging", func(t *testing.T, s release.Store, o Observed) Observed { return o }, PhaseStarting, "none", []string{relTarget}},
299 {"starting after acknowledging", func(t *testing.T, s release.Store, o Observed) Observed { return step(t, s, o, ack(relSource)) }, PhaseStarting, "none", []string{relTarget}},
300 {"releasing before its receipt", func(t *testing.T, s release.Store, o Observed) Observed {
301 o = step(t, s, step(t, s, o, ack(relSource)), ack(relTarget))
302 return step(t, s, o, ToReleasing("rel-1", relObserved))
303 }, PhasePinning, "none", []string{relTarget}},
304 {"releasing after its receipt", func(t *testing.T, s release.Store, o Observed) Observed {
305 o = step(t, s, step(t, s, o, ack(relSource)), ack(relTarget))
306 o = step(t, s, o, ToReleasing("rel-1", relObserved))
307 h := o.Claim.Holder
308 return step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 1, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
309 }, PhaseReleasing, relSource, []string{relSource, relTarget}},
310 } {
311 t.Run(tc.name, func(t *testing.T) {
312 s, o := relServing(t)
313 o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{})
314 if err != nil {
315 t.Fatal(err)
316 }
317 o = tc.prepare(t, s, o)
318 stamp := o.Claim.Holder.Stamp
319 o, err = FenceSource(ctx, s, o, "rel-1", putFence(t, s, relSource), nil, relAuthority{})
320 if err != nil {
321 t.Fatal(err)
322 }
323 c := o.Claim
324 if c.Op.Phase != tc.wantPhase || c.Op.Exec != tc.wantExec || !slices.Equal(c.Op.Participants, tc.wantParts) || c.Holder.Stamp != stamp || !slices.Contains(c.Lost, relSource) {
325 t.Fatalf("phase %s exec %s parts %v stamp %d lost %v", c.Op.Phase, c.Op.Exec, c.Op.Participants, c.Holder.Stamp, c.Lost)
326 }
327 if c.Op.Intent["reason"] != ReasonDrain {
328 t.Fatal("the intent was rewritten")
329 }
330 })
331 }
332}
333
334// relAtPhase starts a drain and drives it to phase. "probing+P" is probing
335// after the first admission opened P.
336func relAtPhase(t *testing.T, phase string) (release.Store, Observed) {
337 t.Helper()
338 ctx := context.Background()
339 s, o := relServing(t)
340 o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{})
341 if err != nil {
342 t.Fatal(err)
343 }
344 o = step(t, s, step(t, s, o, ack(relSource)), ack(relTarget))
345 if phase == PhaseStarting {
346 return s, o
347 }
348 o = step(t, s, o, ToReleasing("rel-1", relObserved))
349 if phase == PhaseReleasing {
350 return s, o
351 }
352 h := o.Claim.Holder
353 o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
354 if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil {
355 t.Fatal(err)
356 }
357 o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved)))
358 if phase == PhasePinning {
359 return s, o
360 }
361 o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1"))
362 if phase == PhaseInstalling {
363 return s, o
364 }
365 o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7)))
366 if phase == PhaseInstalled {
367 return s, o
368 }
369 o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
370 o = step(t, s, o, ToProbing("rel-1", relObserved))
371 if phase == PhaseRelocatingProbing {
372 return s, o
373 }
374 o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
375 Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"}))
376 return s, o
377}
378
379func TestRelocationReplacementTable(t *testing.T) {
380 ctx := context.Background()
381 for _, tc := range []struct {
382 at, wantPhase, wantExec string
383 newStamp, keepP bool
384 }{
385 {PhaseStarting, PhaseStarting, relSource, false, false},
386 {PhaseReleasing, PhaseReleasing, relSource, false, false},
387 {PhasePinning, PhasePinning, "none", false, false},
388 {PhaseInstalling, PhaseInstalling, relSpare, true, false},
389 {PhaseInstalled, PhaseInstalled, relSpare, true, true},
390 {PhaseRelocatingProbing, PhaseInstalled, relSpare, true, true},
391 {"probing+P", PhasePinning, "none", false, false},
392 } {
393 t.Run(tc.at, func(t *testing.T) {
394 s, o := relAtPhase(t, tc.at)
395 before := o.Claim.Holder.Stamp
396 o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{})
397 if err != nil {
398 t.Fatal(err)
399 }
400 c := o.Claim
401 if c.Op.Phase != tc.wantPhase || c.Op.Exec != tc.wantExec || c.Op.Intent["target_box"] != relSpare || !slices.Contains(c.Lost, relTarget) || slices.Contains(c.Op.Participants, relTarget) {
402 t.Fatalf("phase %s exec %s intent %v lost %v parts %v", c.Op.Phase, c.Op.Exec, c.Op.Intent, c.Lost, c.Op.Participants)
403 }
404 if (c.Holder.Stamp != before) != tc.newStamp {
405 t.Fatalf("stamp %d -> %d", before, c.Holder.Stamp)
406 }
407 if tc.keepP && c.Op.Journal["prepared"] != "P1" {
408 t.Fatalf("the unopened P must be reused: %q", c.Op.Journal["prepared"])
409 }
410 if c.Op.Journal["local_install"] != "" {
411 t.Fatal("a replacement inherited its predecessor's receipt")
412 }
413 if tc.wantPhase == PhaseInstalled {
414 if _, err := Update(ctx, s, o, ToProbing("rel-1", relObserved)); err == nil {
415 t.Fatal("probing without the replacement's own local install")
416 }
417 }
418 })
419 }
420}
421
422func TestRelocationTwoReplacementsThenStuck(t *testing.T) {
423 ctx := context.Background()
424 s, o := relAtPhase(t, PhasePinning)
425 var err error
426 for _, next := range []string{relSpare, relSpare2} {
427 old := o.Claim.Op.Intent["target_box"]
428 if o, err = ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, old), nil, Target{next, relEnrollment(next)}, "", relAuthority{}); err != nil {
429 t.Fatal(err)
430 }
431 }
432 if _, err = ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relSpare2), nil, Target{"eve-knitting", relEnrollment("eve-knitting")}, "", relAuthority{}); !errors.Is(err, ErrReplacementsExceeded) {
433 t.Fatalf("third replacement: %v", err)
434 }
435 o = step(t, s, o, MarkStuck("rel-1", "two target replacements used"))
436 if journalN(o.Claim.Op, "replacements") != 2 {
437 t.Fatal("stuck reset the count")
438 }
439 if _, err := Update(ctx, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P9")); !errors.Is(err, ErrRelocation) {
440 t.Fatalf("a stuck operation moved: %v", err)
441 }
442}
443
444// Final review: a malformed replacements count is an error, never 0
445// (absence or malformed data is never evidence). Absent is 0.
446func TestRelocationMalformedReplacementsRefuses(t *testing.T) {
447 ctx := context.Background()
448 s, o := relAtPhase(t, PhasePinning)
449 if n, err := Replacements(o.Claim); err != nil || n != 0 {
450 t.Fatalf("absent: %d %v", n, err)
451 }
452 o = step(t, s, o, func(c *Claim) error { c.Op.Journal["replacements"] = "two"; return nil })
453 if _, err := Replacements(o.Claim); !errors.Is(err, ErrRelocation) {
454 t.Fatalf("malformed count: %v", err)
455 }
456 if _, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{}); !errors.Is(err, ErrRelocation) || errors.Is(err, ErrReplacementsExceeded) {
457 t.Fatalf("a replacement over a malformed count: %v", err)
458 }
459 after, err := Read(ctx, s, o.Host)
460 if err != nil {
461 t.Fatal(err)
462 }
463 if after.ETag != o.ETag || after.Claim.Op.Intent["target_box"] != relTarget {
464 t.Fatalf("the claim moved: %+v", after.Claim.Op)
465 }
466}
467
468func TestRelocationReplacementInReleasingAcknowledgesBeforePinning(t *testing.T) {
469 ctx := context.Background()
470 s, o := relAtPhase(t, PhaseReleasing)
471 h := o.Claim.Holder
472 o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
473 o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{})
474 if err != nil || o.Claim.Op.Phase != PhaseReleasing {
475 t.Fatalf("replacement in releasing: %v %+v", err, o.Claim.Op)
476 }
477 if _, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); !errors.Is(err, ErrRelocation) {
478 t.Fatalf("left releasing before the replacement acknowledged: %v", err)
479 }
480 o = step(t, s, o, ack(relSpare))
481 if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil || o.Claim.Op.Phase != PhasePinning {
482 t.Fatalf("after the acknowledgement: %v %+v", err, o.Claim.Op)
483 }
484}
485
486func TestRelocationExhaustedTargetIsFencedIntoAStuckPinning(t *testing.T) {
487 ctx := context.Background()
488 exhaust := func(t *testing.T, s release.Store, o Observed) Observed {
489 var err error
490 for _, next := range []string{relSpare, relSpare2} {
491 old := o.Claim.Op.Intent["target_box"]
492 if o, err = ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, old), nil, Target{next, relEnrollment(next)}, "", relAuthority{}); err != nil {
493 t.Fatal(err)
494 }
495 }
496 return o
497 }
498 t.Run("before exhaustion it is a replacement", func(t *testing.T) {
499 s, o := relAtPhase(t, PhasePinning)
500 if _, err := FenceExhaustedTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, relAuthority{}); !errors.Is(err, ErrRelocation) {
501 t.Fatalf("fenced as exhausted with replacements left: %v", err)
502 }
503 })
504 // Round 8 finding 1: while the live source is still op.exec, the
505 // target's fence consumes the target and records stuck; the source's
506 // execution identity, phase and release evidence stay.
507 for _, phase := range []string{PhaseStarting, PhaseReleasing} {
508 t.Run("dead while the source executes in "+phase, func(t *testing.T) {
509 s, o := relAtPhase(t, phase)
510 if phase == PhaseReleasing {
511 h := o.Claim.Holder
512 o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
513 }
514 o = exhaust(t, s, o)
515 before := o.Claim
516 ref := putFence(t, s, relSpare2)
517 o, err := FenceExhaustedTarget(ctx, s, o, "rel-1", ref, nil, relAuthority{})
518 if err != nil {
519 t.Fatal(err)
520 }
521 c := o.Claim
522 if c.Op.Phase != phase || c.Op.Exec != relSource || c.Holder.Stamp != before.Holder.Stamp || c.Op.Journal["stuck"] == "" ||
523 !slices.Contains(c.Op.Participants, relSource) || slices.Contains(c.Op.Participants, relSpare2) || !slices.Contains(c.Lost, relSpare2) {
524 t.Fatalf("the live source was revoked: %+v lost %v", c.Op, c.Lost)
525 }
526 for _, k := range []string{"release", "release.phase", "releasing_since", "exec_ack." + relSource} {
527 if c.Op.Journal[k] != before.Op.Journal[k] {
528 t.Fatalf("%s changed: %q -> %q", k, before.Op.Journal[k], c.Op.Journal[k])
529 }
530 }
531 if phase == PhaseReleasing && c.Op.Journal["release"] == "" {
532 t.Fatal("the source's release evidence was dropped")
533 }
534 if l := c.EvidenceRefs[len(c.EvidenceRefs)-1]; l.Box != relSpare2 || l.Fence != ref {
535 t.Fatalf("the dead target's loss: %+v", c.EvidenceRefs)
536 }
537 if err := c.Validate(); err != nil {
538 t.Fatal(err)
539 }
540 })
541 }
542 t.Run("dead after its local install, P unopened", func(t *testing.T) {
543 s, o := relAtPhase(t, PhasePinning)
544 o = exhaust(t, s, o)
545 o = step(t, s, o, ack(relSpare2))
546 o = step(t, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P1"))
547 o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7)))
548 o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
549 if _, err := Update(ctx, s, o, MarkStuck("rel-1", "x")); !errors.Is(err, ErrRelocation) {
550 t.Fatalf("MarkStuck between installed and a probe: %v", err)
551 }
552 ref := putFence(t, s, relSpare2)
553 o, err := FenceExhaustedTarget(ctx, s, o, "rel-1", ref, nil, relAuthority{})
554 if err != nil {
555 t.Fatal(err)
556 }
557 c := o.Claim
558 if c.Op.Phase != PhasePinning || c.Op.Exec != "none" || c.Op.Journal["stuck"] == "" || slices.Contains(c.Op.Participants, relSpare2) || !slices.Contains(c.Lost, relSpare2) ||
559 c.Op.Journal["local_install"] != "" || c.Op.Journal["pin"] == "" || c.Op.Journal["prepared"] != "P1" || c.Op.Journal["prepared_authority"] == "" {
560 t.Fatalf("exhausted and dead: %+v lost %v", c.Op, c.Lost)
561 }
562 if len(c.EvidenceRefs) != 3 || c.EvidenceRefs[2].Box != relSpare2 || c.EvidenceRefs[2].Fence != ref {
563 t.Fatalf("the dead target's loss: %+v", c.EvidenceRefs)
564 }
565 if _, err := Update(ctx, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P2")); !errors.Is(err, ErrRelocation) {
566 t.Fatalf("a stuck operation moved: %v", err)
567 }
568 if _, err := FenceExhaustedTarget(ctx, s, o, "rel-1", ref, nil, relAuthority{}); err == nil {
569 t.Fatal("the fence was consumed twice")
570 }
571 })
572 t.Run("dead after P was opened", func(t *testing.T) {
573 s, o := relAtPhase(t, "probing+P")
574 o = exhaust(t, s, o) // after P's admission each replacement is a nested pinning
575 o = step(t, s, o, ack(relSpare2))
576 o = step(t, s, o, JournalPin("rel-1", relPin(o, 60, 9, relObserved.Add(FailoverPinDelay))))
577 o = step(t, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P3"))
578 o = step(t, s, o, ToInstalled("rel-1", relPrepared("P3", 60, 9)))
579 o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
580 o = step(t, s, o, ToProbing("rel-1", relObserved))
581 o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
582 Box: relSpare2, Mode: "cold", NewGeneration: "P3", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"}))
583 o, err := FenceExhaustedTarget(ctx, s, o, "rel-1", putFence(t, s, relSpare2), nil, relAuthority{})
584 if err != nil {
585 t.Fatal(err)
586 }
587 c := o.Claim
588 if c.Op.Phase != PhasePinning || c.Op.Exec != "none" || c.Op.Journal["stuck"] == "" || c.Op.Journal["pin"] != "" || c.Op.Journal["prepared"] != "" {
589 t.Fatalf("nested stuck pinning: %+v", c.Op)
590 }
591 })
592}
593
594func TestRelocationProbeFailureIsANestedRelocation(t *testing.T) {
595 ctx := context.Background()
596 s, o := relAtPhase(t, "probing+P")
597 o = step(t, s, o, ToProbeFailed("rel-1", relObserved))
598 if o.Claim.Op.Journal["probe_failed_since"] == "" {
599 t.Fatal("probe-failed did not start its release deadline")
600 }
601 h := o.Claim.Holder
602 rel := Release{ReleasedBy: relTarget, Enrollment: relEnrollment(relTarget), Generation: h.Generation, Session: h.Session, FinalTXID: 55, ManifestSeq: 9, Receipt: "r2", ReceiptSHA256: strings.Repeat("d", 64)}
603 o = step(t, s, o, JournalRelease("rel-1", rel))
604 if _, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, &Target{relTarget, relEnrollment(relTarget)}); err == nil {
605 t.Fatal("the failed target was named again")
606 }
607 o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, &Target{relSpare, relEnrollment(relSpare)})
608 if err != nil {
609 t.Fatal(err)
610 }
611 if o.Claim.Op.Phase != PhasePinning || o.Claim.Op.Exec != "none" {
612 t.Fatalf("nested relocation: %+v", o.Claim.Op)
613 }
614 if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 54, 9, relObserved))); !errors.Is(err, ErrRelocation) {
615 t.Fatalf("pin lost the probe's writes: %v", err)
616 }
617 o = step(t, s, o, JournalPin("rel-1", relPin(o, 55, 9, relObserved)))
618 o = step(t, s, o, ack(relSpare))
619 o = toProbing(t, s, o, "P2", 55)
620 // released coverage on P2's opening: the predecessor is the failed
621 // target's probe session, whose release this operation journaled.
622 o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
623 Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"}))
624 o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session}))
625 o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}))
626 o = step(t, s, o, CompleteRelocation("rel-1", counter()))
627 var owed []string
628 for _, ob := range o.Claim.Obligations {
629 owed = append(owed, ob.Box)
630 }
631 slices.Sort(owed)
632 if !slices.Equal(owed, []string{relSource, relTarget}) {
633 t.Fatalf("park-local owed by the source and the abandoned target: %v", owed)
634 }
635}
636
637func TestRelocationAdmissionRules(t *testing.T) {
638 ctx := context.Background()
639 probe := func(o Observed, gen string, certified bool) Admission {
640 return Admission{ObservedOpID: "rel-1", ObservedPhase: o.Claim.Op.Phase, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
641 Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: gen, PredecessorCertified: certified, Region: "ie", PredecessorRegion: "ie"}
642 }
643 for _, tc := range []struct {
644 name, at string
645 admit func(Observed) Admission
646 }{
647 {"the target admits in installing", PhaseInstalling, func(o Observed) Admission { return probe(o, "P1", false) }},
648 {"the target admits in installed", PhaseInstalled, func(o Observed) Admission { return probe(o, "P1", false) }},
649 {"the first probe skips P", PhaseRelocatingProbing, func(o Observed) Admission { return probe(o, "other", false) }},
650 {"P is opened twice", "probing+P", func(o Observed) Admission { return probe(o, "P1", false) }},
651 {"certified after P was opened", "probing+P", func(o Observed) Admission { return probe(o, "P3", true) }},
652 {"released coverage (claim Plan 11 grants none)", PhaseRelocatingProbing, func(o Observed) Admission {
653 a := probe(o, "P1", false)
654 a.Coverage = "released"
655 return a
656 }},
657 {"warm probe", PhaseRelocatingProbing, func(o Observed) Admission {
658 a := probe(o, "", false)
659 a.Mode, a.PredecessorCertified = "warm", true
660 return a
661 }},
662 {"the source certifies in starting", PhaseStarting, func(o Observed) Admission { return probe(o, "g2", true) }},
663 // Claim Plan 11 R1 as accepted: the source's ordinary admission in
664 // starting stays refused until its admission-fence authority exists.
665 {"the source wakes in starting", PhaseStarting, func(o Observed) Admission { return probe(o, "g2", false) }},
666 } {
667 t.Run(tc.name, func(t *testing.T) {
668 s, o := relAtPhase(t, tc.at)
669 if _, err := Update(ctx, s, o, AdmitWriter(tc.admit(o))); !errors.Is(err, ErrWrongState) {
670 t.Fatalf("admitted: %v", err)
671 }
672 })
673 }
674 t.Run("certified names another predecessor", func(t *testing.T) {
675 s, o := relAtPhase(t, PhaseRelocatingProbing)
676 o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1].Session++
677 if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", true))); !errors.Is(err, ErrWrongState) {
678 t.Fatalf("admitted: %v", err)
679 }
680 })
681 t.Run("certified names another predecessor box", func(t *testing.T) {
682 s, o := relAtPhase(t, PhaseRelocatingProbing)
683 o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1].Box = relSpare
684 if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", true))); !errors.Is(err, ErrWrongState) {
685 t.Fatalf("admitted: %v", err)
686 }
687 })
688 t.Run("a third attempt reopens P", func(t *testing.T) {
689 s, o := relAtPhase(t, "probing+P")
690 o = step(t, s, o, AdmitWriter(probe(o, "P3", false))) // attempt two: a fresh generation
691 if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", false))); !errors.Is(err, ErrWrongState) {
692 t.Fatalf("P was opened again: %v", err)
693 }
694 })
695 t.Run("released outside relocating", func(t *testing.T) {
696 s, o := relServing(t)
697 if _, err := Update(ctx, s, o, AdmitWriter(Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relSource, Mode: "cold", NewGeneration: "g2", Coverage: "released", Region: "ie"})); !errors.Is(err, ErrWrongState) {
698 t.Fatalf("admitted: %v", err)
699 }
700 })
701 t.Run("a stuck executor does not probe", func(t *testing.T) {
702 // The same probe opens P on an operation that is not stuck.
703 s, o := relAtPhase(t, PhaseRelocatingProbing)
704 if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", false))); err != nil {
705 t.Fatalf("the probe on a live operation: %v", err)
706 }
707 s, o = relAtPhase(t, PhaseRelocatingProbing)
708 o = step(t, s, o, MarkStuck("rel-1", "operator"))
709 if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", false))); !errors.Is(err, ErrWrongState) {
710 t.Fatalf("admitted: %v", err)
711 }
712 })
713 t.Run("with no P journaled a probe certifies nothing", func(t *testing.T) {
714 s, o := relAtPhase(t, PhaseRelocatingProbing)
715 o = step(t, s, o, func(c *Claim) error { delete(c.Op.Journal, "prepared"); return nil })
716 if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P9", true))); !errors.Is(err, ErrWrongState) {
717 t.Fatalf("a certified probe with no P was admitted: %v", err)
718 }
719 if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P9", false))); err != nil {
720 t.Fatalf("main's uncertified cold probe: %v", err)
721 }
722 })
723}
724
725func TestRelocationEmptyPinNeedsItsEvidence(t *testing.T) {
726 ctx := context.Background()
727 t.Run("a release at txid 0 vouches", func(t *testing.T) {
728 s, o := relAtPhase(t, PhaseReleasing)
729 h := o.Claim.Holder
730 o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 0, ManifestSeq: 0, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
731 o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil)
732 if err != nil {
733 t.Fatal(err)
734 }
735 o = step(t, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved)))
736 o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1"))
737 if _, err := Update(ctx, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 0))); !errors.Is(err, ErrRelocation) {
738 t.Fatalf("an empty pin accepted a database build: %v", err)
739 }
740 if _, err := Update(ctx, s, o, ToInstalled("rel-1", PreparedAuthority{Generation: "P1", FilesSeq: 0})); !errors.Is(err, ErrRelocation) {
741 t.Fatalf("an empty P without its files manifest: %v", err)
742 }
743 step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 0, 0)))
744 })
745 t.Run("a release at txid 40 does not", func(t *testing.T) {
746 s, o := relAtPhase(t, PhaseReleasing)
747 h := o.Claim.Holder
748 o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
749 o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil)
750 if err != nil {
751 t.Fatal(err)
752 }
753 if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved))); !errors.Is(err, ErrRelocation) {
754 t.Fatalf("an empty pin against a release at 40: %v", err)
755 }
756 })
757 t.Run("a failover of a host that had a writer does not", func(t *testing.T) {
758 s, o := relServing(t)
759 ref := putFence(t, s, relSource)
760 o, err := StartRelocation(ctx, s, o, relStart(ReasonFailover, &ref), relAuthority{})
761 if err != nil {
762 t.Fatal(err)
763 }
764 o = step(t, s, step(t, s, o, ack(relTarget)), PinFailover("rel-1"))
765 if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved.Add(FailoverPinDelay)))); !errors.Is(err, ErrRelocation) {
766 t.Fatalf("a listing that found nothing was taken as empty: %v", err)
767 }
768 })
769 t.Run("a failover of a never-woken host does", func(t *testing.T) {
770 s := release.NewDirStore(t.TempDir())
771 o := servingClaim(t, ctx, s) // session 0: no writer was ever admitted
772 ref := putFence(t, s, relSource)
773 o, err := StartRelocation(ctx, s, o, relStart(ReasonFailover, &ref), relAuthority{})
774 if err != nil {
775 t.Fatal(err)
776 }
777 o = step(t, s, step(t, s, o, ack(relTarget)), PinFailover("rel-1"))
778 step(t, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved.Add(FailoverPinDelay))))
779 })
780}
781
782func TestRelocationProbeAttemptsAreBounded(t *testing.T) {
783 ctx := context.Background()
784 s, o := relAtPhase(t, PhaseRelocatingProbing)
785 for i := 0; i < MaxProbeAttempts; i++ {
786 o = step(t, s, o, BeginProbeAttempt("rel-1", relTarget))
787 }
788 if journalN(o.Claim.Op, "probe_attempts."+relTarget) != MaxProbeAttempts {
789 t.Fatalf("count: %v", o.Claim.Op.Journal)
790 }
791 if _, err := Update(ctx, s, o, BeginProbeAttempt("rel-1", relTarget)); !errors.Is(err, ErrProbeAttemptsExhausted) {
792 t.Fatalf("a fourth attempt: %v", err)
793 }
794 if _, err := Update(ctx, s, o, BeginProbeAttempt("rel-1", relSpare)); !errors.Is(err, ErrRelocation) {
795 t.Fatalf("a non-executor counted an attempt: %v", err)
796 }
797 // A replacement target has its own count.
798 o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{})
799 if err != nil {
800 t.Fatal(err)
801 }
802 if journalN(o.Claim.Op, "probe_attempts."+relSpare) != 0 {
803 t.Fatal("the replacement inherited attempts")
804 }
805}
806
807func TestRelocationDischargeCompletesDiscoveryLikeARetirement(t *testing.T) {
808 ctx := context.Background()
809 s, o := relServing(t)
810 o = step(t, s, o, func(c *Claim) error {
811 c.Touched.DiscoveryOp = "disc"
812 c.Obligations = append(c.Obligations, Obligation{ID: "d1", Op: "disc", Kind: "park-local", Box: relSource, Retirement: "r", Owner: "box:" + relSource})
813 return nil
814 })
815 ref := putFence(t, s, relSource)
816 in := relStart(ReasonFailover, &ref)
817 in.Discharge = &LocalDischarge{Qualified: true, Records: []string{"control/box-volumes/x/1.json"}}
818 o, err := StartRelocation(ctx, s, o, in, relAuthority{})
819 if err != nil {
820 t.Fatal(err)
821 }
822 if len(o.Claim.Obligations) != 0 || !o.Claim.Touched.Complete || o.Claim.Touched.DiscoveryOp != "" {
823 t.Fatalf("discharging the final discovery chore: %+v touched %+v", o.Claim.Obligations, o.Claim.Touched)
824 }
825}
826
827func TestRelocationLossRetainsAFormerWritersSession(t *testing.T) {
828 // The source is fenced after the target's probe admission made the
829 // target the writer: the loss still names the source's last session.
830 ctx := context.Background()
831 s, o := relAtPhase(t, "probing+P")
832 before := o.Claim.Holder
833 var sourceAdmission AdmissionRecord
834 for _, a := range before.Admissions {
835 if a.Box == relSource {
836 sourceAdmission = a
837 }
838 }
839 o, err := FenceSource(ctx, s, o, "rel-1", putFence(t, s, relSource), nil, relAuthority{})
840 if err != nil {
841 t.Fatal(err)
842 }
843 l := o.Claim.EvidenceRefs[0]
844 if l.Box != relSource || l.Session != sourceAdmission.Session || l.Generation != sourceAdmission.Generation || l.LastAdmission == nil || !reflect.DeepEqual(*l.LastAdmission, sourceAdmission) ||
845 l.Incarnation != before.Incarnation || l.Stamp != before.Stamp {
846 t.Fatalf("retained: %+v, source admission %+v", l, sourceAdmission)
847 }
848 if err := o.Claim.Validate(); err != nil {
849 t.Fatal(err)
850 }
851}
852
853func TestRelocationRetainsEveryFencedBoxAcrossCompletion(t *testing.T) {
854 // A drain whose target is terminated in installing: the replacement
855 // completes, and the dead target's loss is still on the claim.
856 ctx := context.Background()
857 s, o := relAtPhase(t, PhaseInstalling)
858 ref := putFence(t, s, relTarget)
859 o, err := ReplaceTarget(ctx, s, o, "rel-1", ref, nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{})
860 if err != nil {
861 t.Fatal(err)
862 }
863 o = step(t, s, o, ack(relSpare))
864 o = step(t, s, o, ToInstalled("rel-1", relPrepared("P2", 40, 7)))
865 o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
866 o = step(t, s, o, ToProbing("rel-1", relObserved))
867 o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
868 Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"}))
869 o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session}))
870 o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}))
871 o = step(t, s, o, CompleteRelocation("rel-1", counter()))
872 c := o.Claim
873 if len(c.EvidenceRefs) != 1 || c.EvidenceRefs[0].Box != relTarget || c.EvidenceRefs[0].Role != "target" || c.EvidenceRefs[0].Session != 0 || !slices.Contains(c.LossRefs, c.EvidenceRefs[0].ID) {
874 t.Fatalf("the abandoned target's loss: %+v refs %v", c.EvidenceRefs, c.LossRefs)
875 }
876 if err := c.Validate(); err != nil {
877 t.Fatal(err)
878 }
879}
880
881func TestRelocationStartGuard(t *testing.T) {
882 serving := func(obs ...Obligation) *Claim {
883 return &Claim{Holder: &Holder{Incarnation: "i"}, Op: Op{Kind: KindServing, Exec: relSource}, Obligations: obs}
884 }
885 for _, tc := range []struct {
886 name string
887 ob Obligation
888 fenced bool
889 want error
890 }{
891 {"park-generation in the region (L11)", Obligation{Kind: "park-generation", Region: "ie"}, false, nil},
892 {"park-replica in the region", Obligation{Kind: "park-replica", Region: "ie"}, false, ErrPendingChore},
893 {"park-replica elsewhere", Obligation{Kind: "park-replica", Region: "uk"}, false, nil},
894 {"park-local on the target", Obligation{Kind: "park-local", Box: relTarget}, true, ErrPendingChore},
895 {"park-local on a live source", Obligation{Kind: "park-local", Box: relSource}, false, ErrPendingChore},
896 {"park-local on a fenced source", Obligation{Kind: "park-local", Box: relSource}, true, nil},
897 } {
898 t.Run(tc.name, func(t *testing.T) {
899 if err := RelocationStartGuard(serving(tc.ob), "", "ie", relSource, relTarget, tc.fenced); !errors.Is(err, tc.want) {
900 t.Fatalf("got %v, want %v", err, tc.want)
901 }
902 })
903 }
904}
905
906func TestRelocationLocalInstallReceiptIsBound(t *testing.T) {
907 ctx := context.Background()
908 for _, tc := range []struct {
909 name string
910 mutate func(*LocalInstall)
911 }{
912 {"another box", func(l *LocalInstall) { l.Box = relSpare }},
913 {"another enrollment", func(l *LocalInstall) { l.Enrollment = relEnrollment(relSpare) }},
914 {"a later writer stamp", func(l *LocalInstall) { l.InstallStamp++ }},
915 } {
916 t.Run(tc.name, func(t *testing.T) {
917 s, o := relAtPhase(t, PhaseInstalled)
918 l := localInstall(o.Claim)
919 tc.mutate(&l)
920 if _, err := Update(ctx, s, o, RecordLocalInstall("rel-1", l)); !errors.Is(err, ErrRelocation) {
921 t.Fatalf("recorded: %v", err)
922 }
923 })
924 }
925 s, o := relAtPhase(t, PhaseInstalled)
926 l := localInstall(o.Claim)
927 o = step(t, s, o, RecordLocalInstall("rel-1", l))
928 if _, err := Update(ctx, s, o, RecordLocalInstall("rel-1", l)); !errors.Is(err, ErrAlreadyRecorded) {
929 t.Fatalf("replay: %v", err)
930 }
931 if _, err := Update(ctx, s, o, MarkStuck("rel-1", "operator")); !errors.Is(err, ErrRelocation) {
932 t.Fatalf("stuck between installed and a probe: %v", err)
933 }
934 // B1-2: probing is refused on another box's receipt, even one in the journal.
935 foreign := l
936 foreign.Box = relSpare
937 raw, _ := json.Marshal(foreign)
938 o.Claim.Op.Journal["local_install"] = string(raw)
939 if _, err := Update(ctx, s, o, ToProbing("rel-1", relObserved)); !errors.Is(err, ErrRelocation) {
940 t.Fatalf("probing on another box's receipt: %v", err)
941 }
942}
943
944// Round 5 finding 1: a spare that still owes a local retirement for the
945// host is never named a replacement target, whatever phase the
946// replacement resumes at (installing and installed hand it op.exec in the
947// same CAS); a clean spare is.
948func TestRelocationReplacementRefusesABoxOwingALocalRetirement(t *testing.T) {
949 ctx := context.Background()
950 for _, phase := range []string{PhaseStarting, PhaseInstalling, PhaseInstalled} {
951 t.Run(phase, func(t *testing.T) {
952 s, o := relAtPhase(t, phase)
953 o = step(t, s, o, func(c *Claim) error {
954 c.Obligations = append(c.Obligations, Obligation{ID: "old", Op: "old-op", Kind: "park-local", Box: relSpare, Retirement: "r", Owner: "box:" + relSpare})
955 return nil
956 })
957 ref := putFence(t, s, relTarget)
958 if _, err := ReplaceTarget(ctx, s, o, "rel-1", ref, nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{}); !errors.Is(err, ErrPendingChore) {
959 t.Fatalf("a box owing park-local was named: %v", err)
960 }
961 if _, err := Update(ctx, s, o, NameTarget("rel-1", Target{relSpare, relEnrollment(relSpare)})); err == nil {
962 t.Fatal("NameTarget named a box owing park-local")
963 }
964 o, err := ReplaceTarget(ctx, s, o, "rel-1", ref, nil, Target{relSpare2, relEnrollment(relSpare2)}, "P2", relAuthority{})
965 if err != nil {
966 t.Fatal(err)
967 }
968 c := o.Claim
969 if c.Op.Intent["target_box"] != relSpare2 || slices.Contains(c.Op.Participants, relSpare) || slices.Contains(c.Touched.Boxes, relSpare) {
970 t.Fatalf("the clean spare was not chosen: %+v touched %v", c.Op, c.Touched.Boxes)
971 }
972 if phase != PhaseStarting && c.Op.Exec != relSpare2 {
973 t.Fatalf("exec %s", c.Op.Exec)
974 }
975 })
976 }
977}
978
979// Round 5 finding 2: the automatic fence passes the dead box's regional
980// work (park-generation, park-replica owned by box:<b>) to its region in
981// the same CAS, as DeclareLost does for a declared loss, keeping every
982// other field; another box's work is untouched; local discharge, qualified
983// or not, changes nothing about it.
984func TestRelocationFenceReassignsTheDeadBoxsRegionalWork(t *testing.T) {
985 ctx := context.Background()
986 gen := Obligation{ID: "pg", Op: "old-op", Kind: "park-generation", Region: "ie", Generation: "g0", Retirement: "rg", Owner: "box:" + relSource}
987 rep := Obligation{ID: "pr", Op: "old-op", Kind: "park-replica", Region: "uk", Retirement: "rr", Owner: "box:" + relSource}
988 other := Obligation{ID: "po", Op: "old-op", Kind: "park-generation", Region: "ie", Generation: "g0b", Retirement: "ro", Owner: "box:" + relSpare}
989 local := Obligation{ID: "pl", Op: "old-op", Kind: "park-local", Box: relSource, Retirement: "rl", Owner: "box:" + relSource}
990 for _, qualified := range []bool{false, true} {
991 s, o := relServing(t)
992 o = step(t, s, o, func(c *Claim) error {
993 c.Obligations = append(c.Obligations, gen, rep, other, local)
994 return nil
995 })
996 ref := putFence(t, s, relSource)
997 in := relStart(ReasonFailover, &ref)
998 in.Discharge = &LocalDischarge{Qualified: qualified, Records: []string{"control/box-volumes/x/1.json"}}
999 o, err := StartRelocation(ctx, s, o, in, relAuthority{})
1000 if err != nil {
1001 t.Fatal(err)
1002 }
1003 byID := map[string]Obligation{}
1004 for _, ob := range o.Claim.Obligations {
1005 byID[ob.ID] = ob
1006 }
1007 wantGen, wantRep := gen, rep
1008 wantGen.Owner, wantRep.Owner = "region:ie", "region:uk"
1009 if !reflect.DeepEqual(byID["pg"], wantGen) || !reflect.DeepEqual(byID["pr"], wantRep) || !reflect.DeepEqual(byID["po"], other) {
1010 t.Fatalf("qualified=%v: regional work %+v", qualified, o.Claim.Obligations)
1011 }
1012 if _, owes := byID["pl"]; owes == qualified {
1013 t.Fatalf("qualified=%v: the local chore %+v", qualified, o.Claim.Obligations)
1014 }
1015 want := []ObligationRef{{ID: "pg", Op: "old-op", Retirement: "rg"}, {ID: "pr", Op: "old-op", Retirement: "rr"}}
1016 if len(o.Claim.EvidenceRefs) != 1 || !reflect.DeepEqual(o.Claim.EvidenceRefs[0].Reassigned, want) {
1017 t.Fatalf("qualified=%v: the retained loss does not name the reassignment: %+v", qualified, o.Claim.EvidenceRefs)
1018 }
1019 if err := o.Claim.Validate(); err != nil {
1020 t.Fatal(err)
1021 }
1022 }
1023}
1024
1025// Round 11's sweep: P's opening is journaled with the admission that made
1026// it, so it survives AdmitWriter's trim of holder.admissions to eight. The
1027// removal below stands in for that trim (generation.go drops a covered,
1028// loss-archived head once more than eight are kept); with P's admission
1029// gone from the list, P is still opened: never reopened, and the local
1030// install is never remade.
1031func TestRelocationPOpenedOutlivesTheAdmissionTrim(t *testing.T) {
1032 ctx := context.Background()
1033 s, o := relAtPhase(t, "probing+P")
1034 if o.Claim.Op.Journal["prepared_opened"] != "P1" {
1035 t.Fatalf("the opening was not journaled: %+v", o.Claim.Op.Journal)
1036 }
1037 o = step(t, s, o, func(c *Claim) error {
1038 c.Holder.Admissions = slices.DeleteFunc(c.Holder.Admissions, func(a AdmissionRecord) bool { return a.Generation == "P1" })
1039 return nil
1040 })
1041 if !PreparedOpened(o.Claim) {
1042 t.Fatal("P forgotten with its admission")
1043 }
1044 again := Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
1045 Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"}
1046 if _, err := Update(ctx, s, o, AdmitWriter(again)); !errors.Is(err, ErrWrongState) {
1047 t.Fatalf("P reopened after the trim: %v", err)
1048 }
1049 if _, err := Update(ctx, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))); err == nil {
1050 t.Fatal("the local install was remade after P was opened")
1051 }
1052}
1053
1054// Round 14 finding 1: P's prepared manifest is at sequence 0 whatever the
1055// source's; the pin's sequence (7 here) is its provenance. ToInstalled
1056// refuses a prepared manifest claimed at the source's sequence, and one
1057// built from another source revision, and binds the one it accepts to
1058// the installation stamp.
1059func TestRelocationPreparedManifestIsSequenceZero(t *testing.T) {
1060 ctx := context.Background()
1061 s, o := relAtPhase(t, PhaseInstalling)
1062 var pin RecoveryPin
1063 readJournal(o.Claim.Op, "pin", &pin)
1064 if pin.FilesSeq != 7 {
1065 t.Fatalf("pin at %d", pin.FilesSeq)
1066 }
1067 atSource := relPrepared("P1", 40, 7)
1068 atSource.FilesSeq = 7
1069 if _, err := Update(ctx, s, o, ToInstalled("rel-1", atSource)); !errors.Is(err, ErrRelocation) {
1070 t.Fatalf("a prepared manifest at the source's sequence: %v", err)
1071 }
1072 if _, err := Update(ctx, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 6))); !errors.Is(err, ErrRelocation) {
1073 t.Fatalf("a prepared manifest from another source revision: %v", err)
1074 }
1075 o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7)))
1076 var a PreparedAuthority
1077 readJournal(o.Claim.Op, "prepared_authority", &a)
1078 if a.FilesSeq != 0 || a.SourceFilesSeq != 7 || a.InstallStamp == 0 || a.InstallStamp != journalN(o.Claim.Op, "install_stamp") {
1079 t.Fatalf("journaled authority: %+v", a)
1080 }
1081}
1082
1083// Two-box model finding F3: a target replaced in installing leaves its P
1084// unopened; the replacement builds P'. The serving CAS queues a
1085// park-generation for P in the region, and none for the opened P'.
1086func TestRelocationParksEveryAbandonedPreparedGeneration(t *testing.T) {
1087 ctx := context.Background()
1088 s, o := relAtPhase(t, PhaseInstalling)
1089 o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{})
1090 if err != nil {
1091 t.Fatal(err)
1092 }
1093 var abandoned []string
1094 if !readJournal(o.Claim.Op, "abandoned_prepared", &abandoned) || !slices.Equal(abandoned, []string{"P1"}) {
1095 t.Fatalf("abandoned: %v", abandoned)
1096 }
1097 o = step(t, s, o, ack(relSpare))
1098 o = step(t, s, o, ToInstalled("rel-1", relPrepared("P2", 40, 7)))
1099 o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
1100 o = step(t, s, o, ToProbing("rel-1", relObserved))
1101 o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
1102 Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, PredecessorCertified: true, Region: "ie", PredecessorRegion: "ie"}))
1103 o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session}))
1104 o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}))
1105 o = step(t, s, o, CompleteRelocation("rel-1", counter()))
1106 var parks []string
1107 for _, ob := range o.Claim.Obligations {
1108 if ob.Kind == "park-generation" && ob.Region == "ie" && ob.Owner == "region:ie" {
1109 parks = append(parks, ob.Generation)
1110 }
1111 }
1112 if !slices.Contains(parks, "P1") || slices.Contains(parks, "P2") {
1113 t.Fatalf("park-generation chores: %v (%+v)", parks, o.Claim.Obligations)
1114 }
1115 if err := o.Claim.Validate(); err != nil {
1116 t.Fatal(err)
1117 }
1118}
1119
1120// Round 17 finding 3: the target's probe deadline passes before any
1121// admission (P1 unopened), it releases in probe-failed, then it is
1122// terminated; the replacement's default branch drops P1. It is recorded
1123// first, so the completion of the nested relocation parks it.
1124func TestRelocationParksAPDroppedFromProbeFailed(t *testing.T) {
1125 ctx := context.Background()
1126 s, o := relAtPhase(t, PhaseRelocatingProbing)
1127 if PreparedOpened(o.Claim) {
1128 t.Fatal("P1 was opened")
1129 }
1130 o = step(t, s, o, ToProbeFailed("rel-1", relObserved))
1131 o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{})
1132 if err != nil {
1133 t.Fatal(err)
1134 }
1135 if o.Claim.Op.Phase != PhasePinning || o.Claim.Op.Journal["prepared"] != "" {
1136 t.Fatalf("nested: %+v", o.Claim.Op)
1137 }
1138 var abandoned []string
1139 if !readJournal(o.Claim.Op, "abandoned_prepared", &abandoned) || !slices.Equal(abandoned, []string{"P1"}) {
1140 t.Fatalf("abandoned: %v", abandoned)
1141 }
1142 o = step(t, s, o, ack(relSpare))
1143 o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved.Add(FailoverPinDelay))))
1144 o = toProbing(t, s, o, "P2", 40)
1145 o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseRelocatingProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
1146 Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"}))
1147 o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session}))
1148 o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relSpare, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}))
1149 o = step(t, s, o, CompleteRelocation("rel-1", counter()))
1150 parked := false
1151 for _, ob := range o.Claim.Obligations {
1152 if ob.Kind == "park-generation" && ob.Generation == "P1" && ob.Region == "ie" {
1153 parked = true
1154 }
1155 }
1156 if !parked {
1157 t.Fatalf("P1 was not parked: %+v", o.Claim.Obligations)
1158 }
1159}
1160
1161// Fix round 1: a move converted to relocating journals the source's fence
1162// without its observation time. The claim refuses a pin until FenceSource
1163// has consumed that fence, and then holds it FailoverPinDelay past the
1164// fence's observed_at (H3), as it does for a failover.
1165func TestRelocationConvertedMovePinsOnlyAfterFenceSourceAndTheHold(t *testing.T) {
1166 ctx := context.Background()
1167 s, o, ref, a := c7Fixture(t)
1168 o, err := ConvertMovingToRelocating(ctx, s, o, "move-1", ref, a, &c7Guard{})
1169 if err != nil {
1170 t.Fatal(err)
1171 }
1172 observed, err := time.Parse(time.RFC3339Nano, a.want.ObservedAt)
1173 if err != nil {
1174 t.Fatal(err)
1175 }
1176 pin := func(o Observed, at time.Time) RecoveryPin {
1177 return RecoveryPin{Key: RecoveryPinKey(o.Claim.Host, "move-1", o.Claim.Gen), SHA256: strings.Repeat("a", 64), Generation: "gen-current", MaxTXID: 40,
1178 FilesManifest: "instances/x/files/gen-current/manifest.json", FilesSeq: 7, PinnedAt: at, Release: relRelease}
1179 }
1180 for _, late := range []time.Duration{time.Second, time.Hour} {
1181 if _, err := Update(ctx, s, o, JournalPin("move-1", pin(o, observed.Add(late)))); !errors.Is(err, ErrRelocation) {
1182 t.Fatalf("pinned %s after a fence FenceSource never consumed: %v", late, err)
1183 }
1184 }
1185 if o, err = FenceSource(ctx, s, o, "move-1", ref, nil, a); err != nil {
1186 t.Fatal(err)
1187 }
1188 if o.Claim.Op.Journal["fence.adi-knitting.observed_at"] != a.want.ObservedAt || !slices.Contains(o.Claim.Lost, "adi-knitting") {
1189 t.Fatalf("FenceSource on the converted move: %+v lost %v", o.Claim.Op.Journal, o.Claim.Lost)
1190 }
1191 if _, err := Update(ctx, s, o, JournalPin("move-1", pin(o, observed.Add(FailoverPinDelay-time.Second)))); !errors.Is(err, ErrRelocation) {
1192 t.Fatalf("pinned inside the 30-second window: %v", err)
1193 }
1194 o = step(t, s, o, JournalPin("move-1", pin(o, observed.Add(FailoverPinDelay))))
1195 if !readJournal(o.Claim.Op, "pin", new(RecoveryPin)) {
1196 t.Fatalf("pin not journaled: %+v", o.Claim.Op.Journal)
1197 }
1198}
1199
1200// Task 5 fix round 1: a converted move's own target (in the region the move
1201// was going to) leaves the barrier when NameTarget names the same-region
1202// target, in the same CAS. Otherwise a move target that never acknowledged,
1203// or died, holds the relocation in pinning for ever: nothing in this
1204// region can consume its fence (moves §7.10, §5.8's failover participants).
1205func TestRelocationNameTargetDropsTheConvertedMovesTarget(t *testing.T) {
1206 ctx := context.Background()
1207 s, o, ref, a := c7Fixture(t)
1208 o, err := ConvertMovingToRelocating(ctx, s, o, "move-1", ref, a, &c7Guard{})
1209 if err != nil {
1210 t.Fatal(err)
1211 }
1212 if !slices.Equal(o.Claim.Op.Participants, []string{"bri-knitting"}) || o.Claim.Op.Intent["move_target_box"] != "bri-knitting" {
1213 t.Fatalf("not the converted shape this test is about: %+v", o.Claim.Op)
1214 }
1215 gen := o.Claim.Gen
1216 o = step(t, s, o, NameTarget("move-1", Target{Box: "cai-knitting", Enrollment: strings.Repeat("e", 32)}))
1217 if !slices.Equal(o.Claim.Op.Participants, []string{"cai-knitting"}) || o.Claim.Op.Intent["target_box"] != "cai-knitting" || o.Claim.Gen != gen+1 {
1218 t.Fatalf("the move's target is still on the barrier: %+v", o.Claim.Op)
1219 }
1220 if o.Claim.Op.Intent["move_target_box"] != "bri-knitting" || o.Claim.Op.Journal["replacements"] != "" {
1221 t.Fatalf("the move's history or the replacement count changed: %+v", o.Claim.Op)
1222 }
1223}
1224
1225// R9 (amadan #236): install_release is the journaled pin's release, written
1226// in the installing CAS beside install_stamp, copied again by a re-targeted
1227// installation, and never present without install_stamp.
1228func TestRelocationInstallReleaseFollowsThePin(t *testing.T) {
1229 ctx := context.Background()
1230 pair := func(t *testing.T, c *Claim) {
1231 t.Helper()
1232 _, stamped := c.Op.Journal["install_stamp"]
1233 _, released := c.Op.Journal["install_release"]
1234 if stamped != released {
1235 t.Fatalf("install_stamp present %v, install_release present %v", stamped, released)
1236 }
1237 }
1238 t.Run("the pin needs a release", func(t *testing.T) {
1239 s, o := relAtPhase(t, PhaseReleasing)
1240 h := o.Claim.Holder
1241 o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
1242 o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil)
1243 if err != nil {
1244 t.Fatal(err)
1245 }
1246 bare := relPin(o, 40, 7, relObserved)
1247 bare.Release = ""
1248 if _, err := Update(ctx, s, o, JournalPin("rel-1", bare)); !errors.Is(err, ErrRelocation) {
1249 t.Fatalf("a pin without a release was journaled: %v", err)
1250 }
1251 o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved)))
1252 pair(t, o.Claim)
1253 // A journaled pin that lost its release (malformed) installs nothing.
1254 broken := o
1255 broken.Claim = cloneForTest(t, o.Claim)
1256 broken.Claim.Op.Journal["pin"] = strings.Replace(broken.Claim.Op.Journal["pin"], relRelease, "", 1)
1257 if _, err := Update(ctx, s, broken, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1")); !errors.Is(err, ErrRelocation) {
1258 t.Fatalf("installed without the pin's release: %v", err)
1259 }
1260 })
1261 t.Run("installing copies it", func(t *testing.T) {
1262 _, o := relAtPhase(t, PhaseInstalling)
1263 c := o.Claim
1264 if c.Op.Journal["install_release"] != relRelease || c.Op.Journal["install_stamp"] != strconv.FormatUint(c.Gen, 10) {
1265 t.Fatalf("install_release %q install_stamp %q gen %d", c.Op.Journal["install_release"], c.Op.Journal["install_stamp"], c.Gen)
1266 }
1267 })
1268 for _, at := range []string{PhaseInstalling, PhaseInstalled, PhaseRelocatingProbing} {
1269 t.Run("a re-targeted installation from "+at+" keeps it", func(t *testing.T) {
1270 s, o := relAtPhase(t, at)
1271 stamp := o.Claim.Op.Journal["install_stamp"]
1272 o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{})
1273 if err != nil {
1274 t.Fatal(err)
1275 }
1276 c := o.Claim
1277 if c.Op.Exec != relSpare || c.Op.Journal["install_stamp"] == stamp || c.Op.Journal["install_release"] != relRelease {
1278 t.Fatalf("exec %s install_stamp %q (was %q) install_release %q", c.Op.Exec, c.Op.Journal["install_stamp"], stamp, c.Op.Journal["install_release"])
1279 }
1280 pair(t, c)
1281 })
1282 }
1283 t.Run("a nested pin's release replaces both together", func(t *testing.T) {
1284 s, o := relAtPhase(t, "probing+P")
1285 o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{})
1286 if err != nil {
1287 t.Fatal(err)
1288 }
1289 // Nested pinning: the pin is gone, the last installation's pair is
1290 // kept together (op.exec is none, so nothing can stage under it).
1291 if o.Claim.Op.Phase != PhasePinning || o.Claim.Op.Exec != "none" || o.Claim.Op.Journal["pin"] != "" {
1292 t.Fatalf("not a nested pinning: %+v", o.Claim.Op)
1293 }
1294 pair(t, o.Claim)
1295 next := relPin(o, 40, 7, relObserved.Add(FailoverPinDelay))
1296 next.Release = "v1.5.0"
1297 o = step(t, s, o, JournalPin("rel-1", next))
1298 o = step(t, s, o, ack(relSpare))
1299 o = toProbing(t, s, o, "P3", 40)
1300 if o.Claim.Op.Journal["install_release"] != "v1.5.0" || o.Claim.Op.Journal["install_stamp"] != strconv.FormatUint(o.Claim.Holder.Stamp, 10) {
1301 t.Fatalf("the new installation's pair: %q %q", o.Claim.Op.Journal["install_release"], o.Claim.Op.Journal["install_stamp"])
1302 }
1303 pair(t, o.Claim)
1304 })
1305 t.Run("the serving CAS drops both", func(t *testing.T) {
1306 s, o := relAtPhase(t, "probing+P")
1307 o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session}))
1308 o = step(t, s, o, RecordProbeStop("rel-1", ProbeStop{Box: relTarget, Session: o.Claim.Holder.Session, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}))
1309 o = step(t, s, o, CompleteRelocation("rel-1", counter()))
1310 pair(t, o.Claim)
1311 if o.Claim.Op.Journal["install_stamp"] != "" {
1312 t.Fatalf("serving kept the installation: %+v", o.Claim.Op)
1313 }
1314 })
1315}
1316
1317func cloneForTest(t *testing.T, c *Claim) *Claim {
1318 t.Helper()
1319 out, err := cloneClaim(c)
1320 if err != nil {
1321 t.Fatal(err)
1322 }
1323 return out
1324}
1325
1326// NameTarget in starting is a converted committed landing's alone: a
1327// failover with no executor. A drain in starting never names one.
1328func TestRelocationNameTargetInStartingIsAFailoversAlone(t *testing.T) {
1329 _, o := relAtPhase(t, PhaseStarting)
1330 c := cloneForTest(t, o.Claim)
1331 delete(c.Op.Intent, "target_box")
1332 if err := NameTarget("rel-1", Target{relSpare, relEnrollment(relSpare)})(c); !errors.Is(err, ErrRelocation) {
1333 t.Fatalf("a drain named a target in starting: %v", err)
1334 }
1335}
1336
1337// Stage B final review A3: a repair of the local install replaces the
1338// journaled receipt with the repaired one, in installed and in probing
1339// before P is admitted, only over exactly the journaled receipt and only
1340// for the same box, enrollment and install stamp; never after P opens.
1341func TestReplaceLocalInstallOnlyOverTheJournaledReceiptBeforePOpens(t *testing.T) {
1342 ctx := context.Background()
1343 for _, phase := range []string{PhaseInstalled, PhaseRelocatingProbing} {
1344 s, o := relAtPhase(t, phase)
1345 old := localInstall(o.Claim)
1346 if phase == PhaseInstalled {
1347 o = step(t, s, o, RecordLocalInstall("rel-1", old))
1348 }
1349 repaired := old
1350 repaired.JournalSHA256 = strings.Repeat("c", 64)
1351 other := repaired
1352 other.JournalSHA256 = strings.Repeat("d", 64)
1353 if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", other, repaired)); !errors.Is(err, ErrRelocation) {
1354 t.Fatalf("%s: replaced over a receipt that is not journaled: %v", phase, err)
1355 }
1356 for name, bad := range map[string]func(*LocalInstall){
1357 "another box": func(l *LocalInstall) { l.Box = relSource },
1358 "another enrollment": func(l *LocalInstall) { l.Enrollment = relEnrollment(relSource) },
1359 "another stamp": func(l *LocalInstall) { l.InstallStamp++ },
1360 "a malformed digest": func(l *LocalInstall) { l.JournalSHA256 = "x" },
1361 } {
1362 l := repaired
1363 bad(&l)
1364 if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", old, l)); !errors.Is(err, ErrRelocation) {
1365 t.Fatalf("%s: %s: %v", phase, name, err)
1366 }
1367 }
1368 if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", old, old)); !errors.Is(err, ErrAlreadyRecorded) {
1369 t.Fatalf("%s: the same receipt: %v", phase, err)
1370 }
1371 o = step(t, s, o, ReplaceLocalInstall("rel-1", old, repaired))
1372 var got LocalInstall
1373 if !readJournal(o.Claim.Op, "local_install", &got) || got != repaired || o.Claim.Op.Phase != phase {
1374 t.Fatalf("%s: journaled %+v in %s", phase, got, o.Claim.Op.Phase)
1375 }
1376 if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", old, other)); !errors.Is(err, ErrRelocation) {
1377 t.Fatalf("%s: replaced over the superseded receipt: %v", phase, err)
1378 }
1379 }
1380 s, o := relAtPhase(t, "probing+P")
1381 old := localInstall(o.Claim)
1382 repaired := old
1383 repaired.JournalSHA256 = strings.Repeat("c", 64)
1384 if _, err := Update(ctx, s, o, ReplaceLocalInstall("rel-1", old, repaired)); !errors.Is(err, ErrRelocation) {
1385 t.Fatalf("replaced after P opened: %v", err)
1386 }
1387}
1388