amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostclaim
2
3import (
4 "context"
5 "crypto/ed25519"
6 "encoding/json"
7 "errors"
8 "reflect"
9 "strings"
10 "testing"
11 "time"
12
13 "github.com/carlosframework/platform/internal/release"
14)
15
16// Existing fixtures declare directly in the recovery phase. The end-to-end
17// test below supplies a separate owner for landing -> restoring.
18type testLossRecoveryOwner struct{}
19
20func (testLossRecoveryOwner) ValidateLossRecovery(c Claim, loss LossRecord, in Admission) error {
21 if c.Op.Kind != loss.OpKind || c.Op.Phase != loss.OpPhase || c.Op.Exec != in.Box {
22 return ErrWrongState
23 }
24 return nil
25}
26
27type lossRecoveryOwnerFunc func(Claim, LossRecord, Admission) error
28
29func (f lossRecoveryOwnerFunc) ValidateLossRecovery(c Claim, loss LossRecord, in Admission) error {
30 return f(c, loss, in)
31}
32
33func validLossRecord(t *testing.T) LossRecord {
34 t.Helper()
35 now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
36 inc := strings.Repeat("a", 64)
37 return LossRecord{
38 V: 1, ID: "loss_1", State: "prepared", Actor: "operator", AcceptedAt: now,
39 Host: testHost, HolderIncarnation: inc, HolderStamp: 3, HolderSession: 2, ExpectedClaimGen: 4, ExpectedClaimETag: `"claim"`,
40 OpID: "op", OpKind: KindMoving, OpPhase: "precommit", Box: "box-a", Enrollment: "enroll-a", QuarantineRevision: 9,
41 CloudObservation: LossCloudObservation{ObservedAt: now, Account: "123", Region: "eu-west-1", InstanceID: "i-1", State: "stopped", RequestID: "request-1"},
42 OperatorAttestation: LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true}, FencedAt: now,
43 ConfirmedConsequences: LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(`{"kind":"lost-target"}`), TimersPotentiallyLost: true},
44 Timers: TimerLoss{Knowledge: TimersUnknown},
45 WriteSources: []WriteSource{{Reason: "target", Box: "box-a", Generation: "g1", Session: 3, Region: "ie", Baseline: "snapshot:v1"}},
46 }
47}
48
49func TestLossRecordCreateOnlyConditionalAndImmutable(t *testing.T) {
50 ctx := context.Background()
51 s := release.NewDirStore(t.TempDir())
52 pub, priv, err := ed25519.GenerateKey(nil)
53 if err != nil {
54 t.Fatal(err)
55 }
56 sg := release.RootSigner(priv)
57 record := validLossRecord(t)
58 created, err := CreateLossRecord(ctx, s, sg, record)
59 if err != nil {
60 t.Fatal(err)
61 }
62 if created.ETag == "" {
63 t.Fatal("create returned no ETag")
64 }
65 retry, err := CreateLossRecord(ctx, s, sg, record)
66 if err != nil || retry.ETag != created.ETag {
67 t.Fatalf("idempotent create = %+v, %v", retry, err)
68 }
69 read, err := ReadLossRecord(ctx, s, pub, record.ID)
70 if err != nil || !reflect.DeepEqual(read.Record, record) {
71 t.Fatalf("read = %+v, %v", read, err)
72 }
73
74 committed := read.Record
75 committed.State, committed.ResultClaimGen = "committed", 5
76 next, err := ReplaceLossRecord(ctx, s, sg, read, committed)
77 if err != nil {
78 t.Fatal(err)
79 }
80 mutated := next.Record
81 mutated.FencedAt = mutated.FencedAt.Add(time.Second)
82 if _, err := ReplaceLossRecord(ctx, s, sg, next, mutated); !errors.Is(err, ErrLossRecord) {
83 t.Fatalf("immutable evidence mutation = %v", err)
84 }
85 if _, err := ReplaceLossRecord(ctx, s, sg, read, committed); !errors.Is(err, ErrConflict) {
86 t.Fatalf("stale conditional replace = %v", err)
87 }
88}
89
90func TestTimerKnowledgeNeverConflatesUnknownWithEmptyComplete(t *testing.T) {
91 r := validLossRecord(t)
92 if err := r.Validate(); err != nil {
93 t.Fatal(err)
94 }
95 r.Timers.Knowledge = TimersComplete
96 if err := r.Validate(); !errors.Is(err, ErrLossRecord) {
97 t.Fatalf("complete without source = %v", err)
98 }
99 r.Timers = TimerLoss{}
100 if err := r.Validate(); !errors.Is(err, ErrLossRecord) {
101 t.Fatalf("empty knowledge = %v", err)
102 }
103}
104
105func TestLossRecordCreateRetryUsesSerializedEvidence(t *testing.T) {
106 for _, name := range []string{"ordinary timestamps", "spaced recovery intent", "both"} {
107 t.Run(name, func(t *testing.T) {
108 ctx := t.Context()
109 s := release.NewDirStore(t.TempDir())
110 _, priv, err := ed25519.GenerateKey(nil)
111 if err != nil {
112 t.Fatal(err)
113 }
114 sg := release.RootSigner(priv)
115 record := validLossRecord(t)
116 if name != "spaced recovery intent" {
117 now := time.Now()
118 record.AcceptedAt, record.FencedAt, record.CloudObservation.ObservedAt = now, now, now
119 }
120 if name != "ordinary timestamps" {
121 record.ConfirmedConsequences.RecoveryIntent = json.RawMessage(`{ "kind": "lost-target", "txid": 9007199254740993 }`)
122 }
123 created, err := CreateLossRecord(ctx, s, sg, record)
124 if err != nil {
125 t.Fatal(err)
126 }
127 retry, err := CreateLossRecord(ctx, s, sg, record)
128 if err != nil || retry.ETag != created.ETag {
129 t.Errorf("identical serialized evidence retry = %+v, %v", retry, err)
130 }
131 changed := record
132 changed.FencedAt = changed.FencedAt.Add(time.Nanosecond)
133 if _, err := CreateLossRecord(ctx, s, sg, changed); !errors.Is(err, ErrLossRecord) {
134 t.Fatalf("changed evidence accepted: %v", err)
135 }
136 changed = record
137 changed.ConfirmedConsequences.RecoveryIntent = json.RawMessage(`{"kind":"lost-target","txid":9007199254740992}`)
138 if _, err := CreateLossRecord(ctx, s, sg, changed); !errors.Is(err, ErrLossRecord) {
139 t.Fatalf("changed recovery intent accepted: %v", err)
140 }
141 fresh, err := ReadLossRecord(ctx, s, sg.RootPublicKey(), record.ID)
142 if err != nil || fresh.ETag != created.ETag {
143 t.Fatalf("retry changed stored evidence: %v", err)
144 }
145 })
146 }
147}
148
149func TestAdmissionLossRetentionRequiresExactDurableCopy(t *testing.T) {
150 ctx := context.Background()
151 s := release.NewDirStore(t.TempDir())
152 pub, priv, err := ed25519.GenerateKey(nil)
153 if err != nil {
154 t.Fatal(err)
155 }
156 lost := WritesLost{Box: "target", Generation: "g-target", Session: 4, RemoteMaxTXID: 12}
157 source := WritesLost{Box: "source", Generation: "g-source", Session: 2, RemoteMaxTXID: 10}
158 obs := servingClaim(t, ctx, s)
159 obs, err = Update(ctx, s, obs, func(c *Claim) error {
160 c.LossRefs = []string{"loss_1"}
161 c.Holder.Admissions = []AdmissionRecord{{Region: "ie", Session: 8, LossID: "loss_1", WritesLost: &lost, AdditionalWritesLost: []WritesLost{source}}}
162 return nil
163 })
164 if err != nil {
165 t.Fatal(err)
166 }
167 if err := RequireLossEvidenceArchived(obs.Claim); !errors.Is(err, ErrLossEvidencePending) {
168 t.Fatalf("unarchived evidence = %v", err)
169 }
170 record := validLossRecord(t)
171 record.Host = obs.Host
172 created, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record)
173 if err != nil {
174 t.Fatal(err)
175 }
176 exact := []ArchivedAdmissionLoss{{LossID: "loss_1", AdmissionClaimGen: 8, WritesLost: &lost, AdditionalWritesLost: []WritesLost{source}}}
177 committed := created.Record
178 committed.State, committed.ResultClaimGen, committed.ArchivedAdmissionLosses = "committed", obs.Claim.Gen, exact
179 if _, err := ReplaceLossRecord(ctx, s, release.RootSigner(priv), created, committed); err != nil {
180 t.Fatal(err)
181 }
182 archived, err := ArchiveAdmissionLosses(ctx, s, pub, obs, "loss_1")
183 if err != nil {
184 t.Fatal(err)
185 }
186 if err := RequireLossEvidenceArchived(archived.Claim); err != nil {
187 t.Fatal(err)
188 }
189}
190
191func TestLossBearingAdmissionRequiresEveryFrozenTargetAndSourceCut(t *testing.T) {
192 ctx := context.Background()
193 s := release.NewDirStore(t.TempDir())
194 pub, priv, err := ed25519.GenerateKey(nil)
195 if err != nil {
196 t.Fatal(err)
197 }
198 obs := servingClaim(t, ctx, s)
199 obs, err = Update(ctx, s, obs, func(c *Claim) error {
200 c.Op = Op{Kind: KindReverting, ID: "return", Phase: "recover", Exec: "recovery", Participants: []string{"recovery"}}
201 c.Holder.WriterBox, c.Holder.Generation, c.Holder.Session = "target", "g-target", 4
202 c.LossRefs = []string{"loss_1"}
203 return nil
204 })
205 if err != nil {
206 t.Fatal(err)
207 }
208 earlier := WritesLost{Box: "target", Generation: "g-earlier", Session: 3, RemoteMaxTXID: 7}
209 source := WritesLost{Box: "source", Generation: "g-source", Session: 2, RemoteMaxTXID: 10}
210 in := Admission{ObservedOpID: "return", ObservedPhase: "recover", ObservedSession: 4, ObservedGeneration: "g-target", Box: "recovery", Mode: "cold", NewGeneration: "g-recovery", Region: "ie", PredecessorRegion: "ie", RemoteMaxTXID: 12, AdditionalWritesLost: []WritesLost{earlier, source}}
211 if _, err := Update(ctx, s, obs, AdmitWriter(Admission{ObservedOpID: in.ObservedOpID, ObservedPhase: in.ObservedPhase, ObservedSession: in.ObservedSession, ObservedGeneration: in.ObservedGeneration, Box: in.Box, Mode: in.Mode, NewGeneration: in.NewGeneration, Region: in.Region, PredecessorRegion: in.PredecessorRegion, LossID: "loss_1", AdditionalWritesLost: in.AdditionalWritesLost})); !errors.Is(err, ErrWrongState) {
212 t.Fatalf("direct loss admission = %v", err)
213 }
214 record := validLossRecord(t)
215 record.Host = obs.Host
216 record.HolderIncarnation, record.HolderStamp, record.HolderSession = obs.Claim.Holder.Incarnation, obs.Claim.Holder.Stamp, 4
217 record.OpID, record.OpKind, record.OpPhase, record.Box = "return", KindReverting, "recover", "target"
218 record.WriteSources = []WriteSource{
219 {Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot"},
220 {Reason: "target", Box: "target", Generation: "g-earlier", Session: 3, Region: "ie", Baseline: "earlier-snapshot"},
221 {Reason: "source-cut", Box: "source", Generation: "g-source", Session: 2, Region: "ie", Baseline: "source-seal"},
222 }
223 created, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record)
224 if err != nil {
225 t.Fatal(err)
226 }
227 committed := created.Record
228 committed.State, committed.ResultClaimGen = "committed", obs.Claim.Gen
229 if _, err := ReplaceLossRecord(ctx, s, release.RootSigner(priv), created, committed); err != nil {
230 t.Fatal(err)
231 }
232 validated := []ValidatedWriteSource{
233 {Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot", RemoteMaxTXID: in.RemoteMaxTXID},
234 {Reason: "target", Box: "target", Generation: "g-earlier", Session: 3, Region: "ie", Baseline: "earlier-snapshot", RemoteMaxTXID: earlier.RemoteMaxTXID},
235 {Reason: "source-cut", Box: "source", Generation: "g-source", Session: 2, Region: "ie", Baseline: "source-seal", RemoteMaxTXID: source.RemoteMaxTXID},
236 }
237 for name, mutate := range map[string]func(*Admission, *[]ValidatedWriteSource){
238 "omitted earlier target": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost = a.AdditionalWritesLost[1:] },
239 "omitted earlier target with same txid": func(a *Admission, v *[]ValidatedWriteSource) {
240 (*v)[1].RemoteMaxTXID = a.RemoteMaxTXID
241 a.AdditionalWritesLost = a.AdditionalWritesLost[1:]
242 },
243 "omitted source cut": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost = a.AdditionalWritesLost[:1] },
244 "omitted all additional": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost = nil },
245 "earlier target uses predecessor txid": func(a *Admission, _ *[]ValidatedWriteSource) {
246 a.AdditionalWritesLost[0].RemoteMaxTXID = a.RemoteMaxTXID
247 },
248 "source cut uses predecessor txid": func(a *Admission, _ *[]ValidatedWriteSource) {
249 a.AdditionalWritesLost[1].RemoteMaxTXID = a.RemoteMaxTXID
250 },
251 "predecessor uses earlier txid": func(a *Admission, _ *[]ValidatedWriteSource) { a.RemoteMaxTXID = earlier.RemoteMaxTXID },
252 "duplicate box session with another generation": func(a *Admission, _ *[]ValidatedWriteSource) {
253 duplicate := earlier
254 duplicate.Generation = "another-generation"
255 a.AdditionalWritesLost = append(a.AdditionalWritesLost, duplicate)
256 },
257 "unfrozen tuple": func(a *Admission, _ *[]ValidatedWriteSource) { a.AdditionalWritesLost[0].Session++ },
258 "omitted baseline": func(_ *Admission, v *[]ValidatedWriteSource) { *v = (*v)[:2] },
259 "changed baseline": func(_ *Admission, v *[]ValidatedWriteSource) { (*v)[1].Baseline = "another-snapshot" },
260 } {
261 t.Run(name, func(t *testing.T) {
262 candidate := in
263 candidate.AdditionalWritesLost = append([]WritesLost(nil), in.AdditionalWritesLost...)
264 evidence := append([]ValidatedWriteSource(nil), validated...)
265 mutate(&candidate, &evidence)
266 if _, err := AdmitWriterWithLoss(ctx, s, pub, obs, candidate, "loss_1", evidence, testLossRecoveryOwner{}); !errors.Is(err, ErrLossRecord) {
267 t.Fatalf("invalid declaration admitted: %v", err)
268 }
269 fresh, err := Read(ctx, s, obs.Host)
270 if err != nil || fresh.ETag != obs.ETag {
271 t.Fatalf("refusal changed claim: %v", err)
272 }
273 })
274 }
275 admitted, err := AdmitWriterWithLoss(ctx, s, pub, obs, in, "loss_1", validated, testLossRecoveryOwner{})
276 if err != nil {
277 t.Fatal(err)
278 }
279 last := admitted.Claim.Holder.Admissions[len(admitted.Claim.Holder.Admissions)-1]
280 if last.LossID != "loss_1" || !reflect.DeepEqual(last.AdditionalWritesLost, []WritesLost{earlier, source}) || last.WritesLost == nil || last.WritesLost.RemoteMaxTXID != 12 || last.WritesLost.Session != 4 {
281 t.Fatalf("loss admission = %+v", last)
282 }
283 t.Run("retry validates authorization and evidence", func(t *testing.T) {
284 denied := errors.New("recovery no longer authorized")
285 for _, name := range []string{
286 "identical", "primary txid", "primary txid and baseline", "region", "predecessor region",
287 "observed session", "observed generation", "observed phase", "observed operation",
288 "certified", "coverage", "additional loss", "missing baseline", "changed baseline",
289 "owner denial", "operation", "phase", "executor", "incarnation", "signature", "missing declaration",
290 } {
291 t.Run(name, func(t *testing.T) {
292 candidate := in
293 candidate.AdditionalWritesLost = append([]WritesLost(nil), in.AdditionalWritesLost...)
294 evidence := append([]ValidatedWriteSource(nil), validated...)
295 observation := admitted
296 observation.Claim, err = cloneClaim(admitted.Claim)
297 if err != nil {
298 t.Fatal(err)
299 }
300 key := pub
301 var store release.Store = s
302 switch name {
303 case "primary txid":
304 candidate.RemoteMaxTXID++
305 case "primary txid and baseline":
306 candidate.RemoteMaxTXID++
307 evidence[0].RemoteMaxTXID++
308 case "region":
309 candidate.Region = "uk"
310 case "predecessor region":
311 candidate.PredecessorRegion = "uk"
312 case "observed session":
313 candidate.ObservedSession++
314 case "observed generation":
315 candidate.ObservedGeneration = "other"
316 case "observed phase":
317 candidate.ObservedPhase = "other"
318 case "observed operation":
319 candidate.ObservedOpID = "other"
320 case "certified":
321 candidate.PredecessorCertified = true
322 case "coverage":
323 candidate.Coverage = "sealed"
324 case "additional loss":
325 candidate.AdditionalWritesLost[0].RemoteMaxTXID++
326 case "missing baseline":
327 evidence = nil
328 case "changed baseline":
329 evidence[0].Baseline = "other"
330 case "operation":
331 observation.Claim.Op.ID = "later-operation"
332 candidate.ObservedOpID = "later-operation"
333 case "phase":
334 observation.Claim.Op.Phase = "other"
335 candidate.ObservedPhase = "other"
336 case "executor":
337 observation.Claim.Op.Exec = "other"
338 case "incarnation":
339 observation.Claim.Holder.Incarnation = strings.Repeat("b", 64)
340 case "signature":
341 key = make(ed25519.PublicKey, ed25519.PublicKeySize)
342 case "missing declaration":
343 store = release.NewDirStore(t.TempDir())
344 }
345 calls := 0
346 owner := lossRecoveryOwnerFunc(func(c Claim, loss LossRecord, a Admission) error {
347 calls++
348 if name == "owner denial" {
349 return denied
350 }
351 return (testLossRecoveryOwner{}).ValidateLossRecovery(c, loss, a)
352 })
353 retry, err := AdmitWriterWithLoss(ctx, store, key, observation, candidate, record.ID, evidence, owner)
354 if name == "identical" {
355 if err != nil || retry.ETag != admitted.ETag || !reflect.DeepEqual(retry.Claim, admitted.Claim) || calls != 1 {
356 t.Errorf("identical retry: err %v, owner calls %d, observation %+v", err, calls, retry)
357 }
358 } else if err == nil {
359 t.Errorf("invalid retry accepted")
360 } else if name == "owner denial" && !errors.Is(err, denied) {
361 t.Errorf("owner denial = %v", err)
362 }
363 fresh, err := Read(ctx, s, admitted.Host)
364 if err != nil || fresh.ETag != admitted.ETag {
365 t.Fatalf("retry changed claim: %v", err)
366 }
367 })
368 }
369 })
370}
371
372func TestLossAdmissionRetryInheritsPredecessorRegion(t *testing.T) {
373 for _, region := range []string{"ie", "uk"} {
374 for _, fallback := range []string{"", "fr"} {
375 t.Run(region+"/fallback="+fallback, func(t *testing.T) {
376 ctx := t.Context()
377 s := release.NewDirStore(t.TempDir())
378 pub, priv, err := ed25519.GenerateKey(nil)
379 if err != nil {
380 t.Fatal(err)
381 }
382 obs := servingClaim(t, ctx, s)
383 obs = updateGenerationTest(t, ctx, s, obs, AdmitWriter(Admission{
384 Region: "ie", Box: obs.Claim.Op.Exec, Mode: "cold", NewGeneration: "g-target", PredecessorCertified: true,
385 }))
386 obs = updateGenerationTest(t, ctx, s, obs, func(c *Claim) error {
387 c.Op = Op{Kind: KindReverting, ID: "return", Phase: "recover", Exec: "recovery", Participants: []string{"recovery"}}
388 c.LossRefs = []string{"loss_1"}
389 return nil
390 })
391 h := obs.Claim.Holder
392 record := validLossRecord(t)
393 record.Host, record.HolderIncarnation = obs.Host, h.Incarnation
394 record.HolderStamp, record.HolderSession = h.Stamp, h.Session
395 record.OpID, record.OpKind, record.OpPhase, record.Box = "return", KindReverting, "recover", h.WriterBox
396 record.WriteSources = []WriteSource{{Reason: "target", Box: h.WriterBox, Generation: h.Generation, Session: h.Session, Region: "ie", Baseline: "target-snapshot"}}
397 created, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record)
398 if err != nil {
399 t.Fatal(err)
400 }
401 committed := created.Record
402 committed.State, committed.ResultClaimGen = "committed", obs.Claim.Gen
403 if _, err := ReplaceLossRecord(ctx, s, release.RootSigner(priv), created, committed); err != nil {
404 t.Fatal(err)
405 }
406 in := Admission{ObservedOpID: "return", ObservedPhase: "recover", ObservedSession: h.Session, ObservedGeneration: h.Generation,
407 Box: "recovery", Mode: "cold", NewGeneration: "g-recovery", Region: region, PredecessorRegion: fallback, RemoteMaxTXID: 12}
408 validated := []ValidatedWriteSource{{Reason: "target", Box: h.WriterBox, Generation: h.Generation, Session: h.Session, Region: "ie", Baseline: "target-snapshot", RemoteMaxTXID: 12}}
409 admitted, err := AdmitWriterWithLoss(ctx, s, pub, obs, in, record.ID, validated, testLossRecoveryOwner{})
410 if err != nil {
411 t.Fatal(err)
412 }
413 last := admitted.Claim.Holder.Admissions[len(admitted.Claim.Holder.Admissions)-1]
414 if last.PredecessorRegion != "ie" || last.Region != region {
415 t.Fatalf("admission regions = %q / %q, want %q / %q", last.Region, last.PredecessorRegion, region, "ie")
416 }
417 calls := 0
418 owner := lossRecoveryOwnerFunc(func(c Claim, loss LossRecord, a Admission) error {
419 calls++
420 return (testLossRecoveryOwner{}).ValidateLossRecovery(c, loss, a)
421 })
422 retry, err := AdmitWriterWithLoss(ctx, s, pub, admitted, in, record.ID, validated, owner)
423 if err != nil || retry.ETag != admitted.ETag || !reflect.DeepEqual(retry.Claim, admitted.Claim) || calls != 1 {
424 t.Errorf("identical retry: err %v, owner calls %d, observation %+v", err, calls, retry)
425 }
426 assertAdmissionStored(t, ctx, s, admitted)
427 })
428 }
429 }
430}
431
432func TestTargetOnlyLossAdmissionRequiresColdUncoveredRecovery(t *testing.T) {
433 ctx := context.Background()
434 s := release.NewDirStore(t.TempDir())
435 pub, priv, err := ed25519.GenerateKey(nil)
436 if err != nil {
437 t.Fatal(err)
438 }
439 obs := servingClaim(t, ctx, s)
440 obs, err = Update(ctx, s, obs, func(c *Claim) error {
441 c.Op = Op{Kind: KindReverting, ID: "return", Phase: "recover", Exec: "recovery", Participants: []string{"recovery"}}
442 c.Holder.WriterBox, c.Holder.Generation, c.Holder.Session = "target", "g-target", 4
443 c.LossRefs = []string{"loss_1"}
444 return nil
445 })
446 if err != nil {
447 t.Fatal(err)
448 }
449 record := validLossRecord(t)
450 record.Host, record.HolderIncarnation = obs.Host, obs.Claim.Holder.Incarnation
451 record.HolderStamp, record.HolderSession = obs.Claim.Holder.Stamp, 4
452 record.OpID, record.OpKind, record.OpPhase, record.Box = "return", KindReverting, "recover", "target"
453 record.WriteSources = []WriteSource{{Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot"}}
454 created, err := CreateLossRecord(ctx, s, release.RootSigner(priv), record)
455 if err != nil {
456 t.Fatal(err)
457 }
458 committed := created.Record
459 committed.State, committed.ResultClaimGen = "committed", obs.Claim.Gen
460 if _, err := ReplaceLossRecord(ctx, s, release.RootSigner(priv), created, committed); err != nil {
461 t.Fatal(err)
462 }
463 base := Admission{ObservedOpID: "return", ObservedPhase: "recover", ObservedSession: 4, ObservedGeneration: "g-target", Box: "recovery", Mode: "cold", NewGeneration: "g-recovery", Region: "ie", PredecessorRegion: "ie", RemoteMaxTXID: 12}
464 validated := []ValidatedWriteSource{{Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot", RemoteMaxTXID: 12}}
465 for name, mutate := range map[string]func(*Admission){
466 "warm": func(a *Admission) { a.Mode, a.NewGeneration = "warm", "" },
467 "certified": func(a *Admission) { a.PredecessorCertified = true },
468 "sealed": func(a *Admission) { a.Coverage = "sealed" },
469 } {
470 t.Run(name, func(t *testing.T) {
471 in := base
472 mutate(&in)
473 if _, err := AdmitWriterWithLoss(ctx, s, pub, obs, in, "loss_1", validated, testLossRecoveryOwner{}); !errors.Is(err, ErrLossRecord) {
474 t.Fatalf("unsafe target-loss mode = %v", err)
475 }
476 })
477 }
478 fresh, err := Read(ctx, s, obs.Host)
479 if err != nil || fresh.Claim.Gen != obs.Claim.Gen {
480 t.Fatalf("refusals mutated claim: gen %d, err %v", fresh.Claim.Gen, err)
481 }
482}
483
484func TestLossDeclarationHandoverAdmission(t *testing.T) {
485 ctx := context.Background()
486 s := release.NewDirStore(t.TempDir())
487 pub, priv, err := ed25519.GenerateKey(nil)
488 if err != nil {
489 t.Fatal(err)
490 }
491 sg := release.RootSigner(priv)
492 obs := servingClaim(t, ctx, s)
493 obs, err = Update(ctx, s, obs, func(c *Claim) error {
494 c.Op = Op{Kind: KindLanding, ID: "move", Phase: "probe", Exec: "target", Participants: []string{"target", "recovery"}}
495 c.Holder.WriterBox, c.Holder.Generation, c.Holder.Session = "target", "g-target", 4
496 return nil
497 })
498 if err != nil {
499 t.Fatal(err)
500 }
501 record := validLossRecord(t)
502 record.Host, record.HolderIncarnation = obs.Host, obs.Claim.Holder.Incarnation
503 record.HolderStamp, record.HolderSession = obs.Claim.Holder.Stamp, obs.Claim.Holder.Session
504 record.ExpectedClaimGen, record.ExpectedClaimETag = obs.Claim.Gen, obs.ETag
505 record.OpID, record.OpKind, record.OpPhase, record.Box = "move", KindLanding, "probe", "target"
506 record.WriteSources = []WriteSource{
507 {Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot"},
508 {Reason: "source-cut", Box: "source", Generation: "g-source", Session: 2, Region: "ie", Baseline: "source-seal"},
509 }
510 prepared, err := CreateLossRecord(ctx, s, sg, record)
511 if err != nil {
512 t.Fatal(err)
513 }
514 declared, err := DeclareLost(ctx, s, pub, obs, record.ID, testLossOwner{})
515 if err != nil {
516 t.Fatal(err)
517 }
518 if declared.Claim.Op.Kind != KindLanding || declared.Claim.Op.Phase != "probe" || declared.Claim.Op.Exec != "none" ||
519 declared.Claim.Holder.Stamp != record.HolderStamp || declared.Claim.Holder.Session != record.HolderSession {
520 t.Fatalf("declaration changed recovery authority: %+v", declared.Claim)
521 }
522 committed := prepared.Record
523 committed.State, committed.ResultClaimGen = "committed", declared.Claim.Gen
524 if _, err := ReplaceLossRecord(ctx, s, sg, prepared, committed); err != nil {
525 t.Fatal(err)
526 }
527 // The operation owner hands execution to a new installation in the same
528 // operation, preserving the original declaration and writer provenance.
529 recoveryStamp := declared.Claim.Gen + 1
530 handover, err := Update(ctx, s, declared, func(c *Claim) error {
531 if c.Op.Kind != KindLanding || c.Op.Phase != "probe" || c.Op.Exec != "none" || !lossRefPresent(c, record.ID) {
532 return ErrWrongState
533 }
534 c.Op.Kind, c.Op.Phase, c.Op.Exec = KindRestoring, "restore", "recovery"
535 c.Op.Journal["recovery.loss"] = record.ID
536 c.Holder.Stamp = recoveryStamp
537 return nil
538 })
539 if err != nil {
540 t.Fatal(err)
541 }
542 source := WritesLost{Box: "source", Generation: "g-source", Session: 2, RemoteMaxTXID: 10}
543 in := Admission{ObservedOpID: "move", ObservedPhase: "restore", ObservedSession: 4, ObservedGeneration: "g-target", Box: "recovery", Mode: "cold", NewGeneration: "g-recovery", Region: "ie", PredecessorRegion: "ie", RemoteMaxTXID: 12, AdditionalWritesLost: []WritesLost{source}}
544 validated := []ValidatedWriteSource{
545 {Reason: "target", Box: "target", Generation: "g-target", Session: 4, Region: "ie", Baseline: "target-snapshot", RemoteMaxTXID: 12},
546 {Reason: "source-cut", Box: "source", Generation: "g-source", Session: 2, Region: "ie", Baseline: "source-seal", RemoteMaxTXID: 10},
547 }
548 owner := lossRecoveryOwnerFunc(func(c Claim, loss LossRecord, a Admission) error {
549 if loss.OpKind != KindLanding || loss.OpPhase != "probe" || c.Op.Kind != KindRestoring || c.Op.Phase != "restore" ||
550 c.Op.Exec != "recovery" || a.Box != c.Op.Exec || c.Holder.Stamp != recoveryStamp ||
551 c.Op.Journal["recovery.loss"] != loss.ID || c.Op.Journal["loss."+loss.ID] != `{"kind":"lost-target"}` {
552 return ErrWrongState
553 }
554 // A policy callback cannot rewrite any input used by the admission.
555 c.Holder.Incarnation = "changed"
556 c.Op.Journal["recovery.loss"] = "changed"
557 loss.WriteSources[0].Baseline = "changed"
558 a.AdditionalWritesLost[0].RemoteMaxTXID = 999
559 return nil
560 })
561 if _, err := AdmitWriterWithLoss(ctx, s, pub, declared, in, record.ID, validated, owner); !errors.Is(err, ErrWrongState) {
562 t.Fatalf("admitted before handover: %v", err)
563 }
564 if _, err := AdmitWriterWithLoss(ctx, s, pub, handover, in, record.ID, validated, nil); !errors.Is(err, ErrLossOwnerRequired) {
565 t.Fatalf("admitted without owner: %v", err)
566 }
567 for name, mutate := range map[string]func(*Claim){
568 "kind": func(c *Claim) { c.Op.Kind = KindReverting },
569 "phase": func(c *Claim) { c.Op.Phase = "unauthorized" },
570 "executor": func(c *Claim) { c.Op.Exec = "other" },
571 "stamp": func(c *Claim) { c.Holder.Stamp++ },
572 "handover": func(c *Claim) { delete(c.Op.Journal, "recovery.loss") },
573 "intent": func(c *Claim) { delete(c.Op.Journal, "loss."+record.ID) },
574 "operation": func(c *Claim) { c.Op.ID = "other" },
575 "incarnation": func(c *Claim) { c.Holder.Incarnation = "other" },
576 "loss ref": func(c *Claim) { c.LossRefs = nil },
577 } {
578 t.Run(name, func(t *testing.T) {
579 candidate := handover
580 candidate.Claim, err = cloneClaim(handover.Claim)
581 if err != nil {
582 t.Fatal(err)
583 }
584 mutate(candidate.Claim)
585 if _, err := AdmitWriterWithLoss(ctx, s, pub, candidate, in, record.ID, validated, owner); err == nil {
586 t.Fatal("admitted with changed binding")
587 }
588 })
589 }
590 stale := in
591 stale.ObservedPhase = "probe"
592 if _, err := AdmitWriterWithLoss(ctx, s, pub, handover, stale, record.ID, validated, owner); !errors.Is(err, ErrWrongState) {
593 t.Fatalf("stale admission phase = %v", err)
594 }
595 fresh, err := Read(ctx, s, obs.Host)
596 if err != nil || fresh.ETag != handover.ETag {
597 t.Fatalf("refusal changed claim: %v", err)
598 }
599 admitted, err := AdmitWriterWithLoss(ctx, s, pub, handover, in, record.ID, validated, owner)
600 if err != nil {
601 t.Fatal(err)
602 }
603 last := admitted.Claim.Holder.Admissions[len(admitted.Claim.Holder.Admissions)-1]
604 if last.LossID != record.ID || last.WritesLost == nil || *last.WritesLost != (WritesLost{Box: "target", Generation: "g-target", Session: 4, RemoteMaxTXID: 12}) ||
605 !reflect.DeepEqual(last.AdditionalWritesLost, []WritesLost{source}) || admitted.Claim.Holder.WriterBox != "recovery" ||
606 admitted.Claim.Op.ID != record.OpID || admitted.Claim.Op.Kind != KindRestoring || admitted.Claim.Op.Phase != "restore" ||
607 admitted.Claim.Holder.Incarnation != record.HolderIncarnation || admitted.Claim.Op.Journal["recovery.loss"] != record.ID {
608 t.Fatalf("recovery admission = %+v", admitted.Claim)
609 }
610 frozen, err := ReadLossRecord(ctx, s, pub, record.ID)
611 if err != nil || !reflect.DeepEqual(frozen.Record, committed) {
612 t.Fatalf("recovery rewrote frozen declaration: %+v, %v", frozen.Record, err)
613 }
614}
615
616func TestLegacyLossWithoutDeclarationRetainsEvidence(t *testing.T) {
617 a := AdmissionRecord{Region: "ie", Session: 3, WritesLost: &WritesLost{Box: "legacy", Session: 0}}
618 if !validWriteLosses(a.WritesLost, nil) {
619 t.Fatal("legacy zero-session loss no longer parses")
620 }
621 c := &Claim{Holder: &Holder{Admissions: []AdmissionRecord{a}}}
622 if err := RequireLossEvidenceArchived(c); !errors.Is(err, ErrLossEvidencePending) {
623 t.Fatalf("unarchived legacy loss discarded: %v", err)
624 }
625}
626