amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostops
2
3import (
4 "context"
5 "crypto/ed25519"
6 "crypto/rand"
7 "encoding/json"
8 "errors"
9 "reflect"
10 "strings"
11 "sync"
12 "testing"
13 "time"
14
15 "github.com/carlosframework/platform/internal/hostactors"
16 "github.com/carlosframework/platform/internal/hostclaim"
17 "github.com/carlosframework/platform/internal/hostfence"
18 "github.com/carlosframework/platform/internal/hostresolve"
19 "github.com/carlosframework/platform/internal/moves"
20 "github.com/carlosframework/platform/internal/provision"
21 "github.com/carlosframework/platform/internal/release"
22 "github.com/superfly/ltx"
23)
24
25// J4's moves owner and the J17 declare-lost grant: a console-initiated
26// declaration that a move's target box is lost (moves spec §7.10), through
27// the generic hostresolve service with only the move kinds' owner
28// registered, writing only through the typed ControlWriter's private
29// scoped store.
30
31const (
32 lossOp = "mv_abcdefghijklmnopqrstuvwxyz"
33 lossHost = "shop.example"
34 lossBucket = "bucket"
35)
36
37// lossFaultStore fails the first Put of one key, before or after it lands.
38type lossFaultStore struct {
39 release.Store
40 mu sync.Mutex
41 key string
42 failBefore bool
43 failAfter bool
44 puts []string
45}
46
47func (s *lossFaultStore) Put(ctx context.Context, key string, body []byte, match string) (string, error) {
48 s.mu.Lock()
49 s.puts = append(s.puts, key)
50 before, after := s.key == key && s.failBefore, s.key == key && s.failAfter
51 if s.key == key {
52 s.failBefore, s.failAfter = false, false
53 }
54 s.mu.Unlock()
55 if before {
56 return "", errors.New("injected put failure")
57 }
58 tag, err := s.Store.Put(ctx, key, body, match)
59 if err == nil && after {
60 return "", errors.New("injected lost answer")
61 }
62 return tag, err
63}
64
65type lossWorld struct {
66 raw *lossFaultStore
67 pub ed25519.PublicKey
68 signer release.Signer
69 control *ControlWriter
70 clock time.Time
71 inc string
72 key string
73 claimK string
74}
75
76// newLossWorld is a host mid-move from ie-box (ie) to uk-box (uk): state
77// "landing" (the target probing, executing), "releasing" (a rollback the
78// target has not yet released), "released-dirty" (the target released
79// dirty and has not returned) or "returning". The target admitted two
80// sessions since the cut (20, 25); the source's cut session is 15 and the
81// seal did not cover the source's own replica.
82func newLossWorld(t *testing.T, state string, mutate func(*hostclaim.Claim, *provision.Record, *AppAdmission)) *lossWorld {
83 t.Helper()
84 ctx := context.Background()
85 pub, priv, err := ed25519.GenerateKey(rand.Reader)
86 if err != nil {
87 t.Fatal(err)
88 }
89 w := &lossWorld{raw: &lossFaultStore{Store: release.NewDirStore(t.TempDir())}, pub: pub, signer: release.RootSigner(priv), clock: time.Now().UTC().Truncate(time.Second)}
90 if w.control, err = NewControlWriter(w.raw, pub, func() time.Time { return w.clock }); err != nil {
91 t.Fatal(err)
92 }
93 d := hostactors.Directory{Store: w.raw, PublicKey: pub, Bucket: lossBucket}
94 if err := d.Initialize(ctx, w.signer); err != nil {
95 t.Fatal(err)
96 }
97 o, err := d.Read(ctx)
98 if err != nil {
99 t.Fatal(err)
100 }
101 r := hostactors.Clone(o.Roster)
102 r.Boxes["ie-box"] = hostactors.Box{Enrollment: strings.Repeat("a", 32), Namespace: "platform", BoxID: "ie-box", Label: "ie-label", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-ie"}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"}
103 r.Boxes["uk-box"] = hostactors.Box{Enrollment: strings.Repeat("b", 32), Namespace: "platform", BoxID: "uk-box", Label: "uk-label", Region: "uk", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-2", InstanceID: "i-uk"}, EvidenceSHA256: strings.Repeat("d", 64), State: "active"}
104 if _, err := d.Replace(ctx, w.signer, o, r); err != nil {
105 t.Fatal(err)
106 }
107 created := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
108 if w.inc, err = hostclaim.Incarnation("bab", "app", lossHost, created); err != nil {
109 t.Fatal(err)
110 }
111 w.key = provision.RecordKey("bab", "app", lossHost)
112 at := w.clock.Add(-time.Hour)
113 seal := moves.Seal{V: moves.SealVersion, N: 42, FromDB: true, SrcCovered: false, SealedAt: at, LeaseUntil: at.Add(time.Minute),
114 Replica: []moves.SealedObject{{Key: "moves/" + lossHost + "/" + lossOp + "/replica/0009/" + ltx.FormatFilename(1, 42), VersionID: "v1", Size: int64(ltx.HeaderSize)}},
115 Files: []moves.SealedObject{{Key: "moves/" + lossHost + "/" + lossOp + "/files/archive", VersionID: "v2", Size: 10}}}
116 c := hostclaim.Claim{V: 2, Host: lossHost, Gen: 30,
117 Holder: &hostclaim.Holder{Family: "instance", Key: w.key, Account: "bab", App: "app", Acquisition: "acq", CreatedAt: created, Incarnation: w.inc,
118 Location: "instances/" + lossHost, Generation: "mv2", Stamp: 21, Session: 25, WriterBox: "uk-box",
119 Admissions: []hostclaim.AdmissionRecord{
120 {Session: 15, Box: "ie-box", Generation: "src", Predecessor: "older", Region: "ie", PredecessorRegion: "ie", Mode: "cold"},
121 {Session: 20, Box: "uk-box", Generation: "mv", Predecessor: "src", Region: "uk", PredecessorRegion: "ie", Mode: "cold", Coverage: "sealed"},
122 {Session: 25, Box: "uk-box", Generation: "mv2", Predecessor: "mv", Region: "uk", PredecessorRegion: "uk", Mode: "cold"},
123 }},
124 Op: hostclaim.Op{Kind: hostclaim.KindLanding, ID: lossOp, Phase: "probing", Exec: "uk-box", Participants: []string{"ie-box", "uk-box"},
125 Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box", "app_id": "canonical", "admission_id": lossOp,
126 "source_enrollment": strings.Repeat("a", 32), "target_enrollment": strings.Repeat("b", 32)},
127 Journal: map[string]string{hostclaim.JournalSeal: seal.Encode(), hostclaim.JournalCutSession: "15", hostclaim.JournalSourceStamp: "15", hostclaim.JournalSrcGen: "src",
128 hostclaim.JournalEvidenceSource: "moves/bab/app/" + lossHost + "/" + lossOp + "/stopped-source.json"}}}
129 flipped, deadline := at, at.Add(2*time.Hour)
130 rec := provision.Record{Schema: 1, Host: lossHost, Slug: "app", Account: "bab", App: "app", Acquisition: "acq", Channel: "edge", Region: "uk", CreatedAt: created,
131 Move: &provision.MoveState{ID: lossOp, Incarnation: w.inc, From: "ie", To: "uk", Cut: "auto", State: provision.MoveLanding, CutPhase: provision.CutFlipped,
132 FlippedAt: &flipped, LandingDeadline: &deadline, LandingPhase: provision.LandProbing}}
133 rb := "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb"
134 requested, release15 := at.Add(time.Minute), at.Add(16*time.Minute)
135 switch state {
136 case "landing":
137 case "releasing", "released-dirty", "returning":
138 c.Op.Kind, c.Op.Phase = hostclaim.KindRestoring, hostclaim.PhaseReleasing
139 c.Op.Intent["rollback"] = rb
140 c.Op.Journal[hostclaim.JournalTargetStamp] = "21"
141 rec.Region = "ie"
142 rec.Move.State = provision.MoveRollingBack
143 rec.Move.Rollback = &provision.MoveRollback{ID: rb, Reason: provision.RollbackProbeAttempts, TargetPhase: provision.RbTargetReleasing, SourcePhase: provision.RbSourceWaiting, RequestedAt: &requested, ReleaseDeadline: &release15}
144 if state != "releasing" {
145 c.Op.Phase, c.Op.Exec = hostclaim.PhaseReleased, ""
146 c.Op.Journal[hostclaim.JournalEvidenceTarget] = "moves/bab/app/" + lossHost + "/" + rb + "/stopped-target.json"
147 c.Obligations = []hostclaim.Obligation{{ID: "loc", Op: lossOp, Kind: "park-replica", Region: "uk", Retirement: "r-loc", Owner: "region:uk",
148 Park: &hostclaim.ParkScope{Location: "instances/" + lossHost, Incarnation: w.inc, EvidenceAt: at, RequestLimit: time.Minute}}}
149 rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = provision.RbTargetReleased, true
150 if state == "returning" {
151 rec.Move.Rollback.TargetPhase = provision.RbTargetReturning
152 }
153 }
154 default:
155 t.Fatalf("state %q", state)
156 }
157 adm := AppAdmission{V: 1, AppID: "canonical", Epoch: 1, Revision: 1, State: "open", Effects: map[string]Effect{}, Registrations: map[string]Registration{lossOp: {Kind: RegistrationMove, Host: lossHost, Owner: c7Owner, AppID: "canonical", Epoch: 1, Claim: SavedClaim{ETag: "serving", Gen: 1}, Expires: w.clock.Add(RegistrationTTL), CreatedAt: w.clock, Incarnation: w.inc}}}
158 if mutate != nil {
159 mutate(&c, &rec, &adm)
160 }
161 if err := c.Validate(); err != nil {
162 t.Fatal(err)
163 }
164 cb, _ := json.Marshal(c)
165 w.claimK, _ = hostclaim.Key(lossHost)
166 if _, err := w.raw.Store.Put(ctx, w.claimK, cb, ""); err != nil {
167 t.Fatal(err)
168 }
169 rbody, _ := json.Marshal(rec)
170 if _, err := w.raw.Store.Put(ctx, w.key, rbody, ""); err != nil {
171 t.Fatal(err)
172 }
173 ab, _ := json.Marshal(adm)
174 if _, err := w.raw.Store.Put(ctx, "apps/bab/app/admission.json", ab, ""); err != nil {
175 t.Fatal(err)
176 }
177 return w
178}
179
180func (w *lossWorld) claim(t *testing.T) hostclaim.Observed {
181 t.Helper()
182 o, err := hostclaim.Read(context.Background(), w.raw, lossHost)
183 if err != nil {
184 t.Fatal(err)
185 }
186 return o
187}
188
189func (w *lossWorld) record(t *testing.T) (provision.Record, string) {
190 t.Helper()
191 rec, tag, err := provision.GetRecordETag(context.Background(), w.raw, "bab", "app", lossHost)
192 if err != nil {
193 t.Fatal(err)
194 }
195 return *rec, tag
196}
197
198func (w *lossWorld) roster(t *testing.T) hostactors.Observed {
199 t.Helper()
200 o, err := hostactors.Directory{Store: w.raw, PublicKey: w.pub, Bucket: lossBucket}.Read(context.Background())
201 if err != nil {
202 t.Fatal(err)
203 }
204 return o
205}
206
207func (w *lossWorld) inspect(t *testing.T, box string) hostresolve.Inspection {
208 t.Helper()
209 in, err := w.control.InspectMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, box)
210 if err != nil {
211 t.Fatal(err)
212 }
213 return in
214}
215
216func (w *lossWorld) submission(in hostresolve.Inspection) hostfence.Submission {
217 w.clock = w.clock.Add(2 * time.Second)
218 return hostfence.Submission{Challenge: in.Challenge, Confirmation: in.Details.ID,
219 Observation: hostfence.CloudObservation{ObservedAt: in.Details.IssuedAt.Add(time.Second), Account: "123", Region: "eu-west-2", InstanceID: "i-uk", State: "stopped", RequestID: "request"},
220 Attestation: hostfence.OperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, Disks: in.Details.Consequences.DestroyedStorageAssertion}}
221}
222
223func (w *lossWorld) declare(sub hostfence.Submission) (hostclaim.Observed, error) {
224 return w.control.DeclareMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub)
225}
226
227func (w *lossWorld) acceptances(t *testing.T) []string {
228 t.Helper()
229 keys, err := w.raw.List(context.Background(), "control/host-loss-acceptances/")
230 if err != nil {
231 t.Fatal(err)
232 }
233 return keys
234}
235
236func lossSnapshot(t *testing.T, w *lossWorld) string {
237 t.Helper()
238 c := w.claim(t)
239 _, tag := w.record(t)
240 return c.ETag + "|" + tag + "|" + w.roster(t).ETag + "|" + strings.Join(w.acceptances(t), ",")
241}
242
243// The landing's target is lost: the declaration rolls the record back clean
244// with the loss as its evidence, takes the target out of the claim, and
245// releases it in the claim with the target region's parks, in a crash-safe
246// sequence whose signed record froze every lost write.
247func TestJ4DeclareLostLandingTarget(t *testing.T) {
248 ctx := context.Background()
249 w := newLossWorld(t, "landing", nil)
250 in := w.inspect(t, "uk-label")
251 if in.Details.Box != "uk-box" || in.Details.Op.Kind != hostclaim.KindLanding || in.Details.Consequences.Transition != moveLossTransition ||
252 !in.Details.Consequences.PossibleWriteLoss || !in.Details.Consequences.PossibleTimerLoss || len(in.Details.Consequences.DestroyedStorageAssertion) != 0 {
253 t.Fatalf("inspection = %+v", in.Details)
254 }
255 _, tag := w.record(t)
256 if in.Details.Record.Key != w.key || in.Details.Record.ETag != tag {
257 t.Fatalf("record observation = %+v", in.Details.Record)
258 }
259 before := w.claim(t)
260 sub := w.submission(in)
261 after, err := w.declare(sub)
262 if err != nil {
263 t.Fatal(err)
264 }
265 id := in.Details.ID
266 lossRef, _ := hostclaim.LossRecordKey(id)
267 loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, id)
268 if err != nil || loss.Record.State != "committed" || loss.Record.OpKind != hostclaim.KindLanding || loss.Record.ExpectedClaimETag != before.ETag {
269 t.Fatalf("loss record = %+v %v", loss.Record, err)
270 }
271 wantSources := []hostclaim.WriteSource{
272 {Reason: "target", Box: "uk-box", Generation: "mv", Session: 20, Region: "uk", Baseline: "replica:uk/mv"},
273 {Reason: "target", Box: "uk-box", Generation: "mv2", Session: 25, Region: "uk", Baseline: "replica:uk/mv2"},
274 {Reason: "source-cut", Box: "ie-box", Generation: "src", Session: 15, Region: "ie", Baseline: "seal:" + lossOp},
275 }
276 if !reflect.DeepEqual(loss.Record.WriteSources, wantSources) {
277 t.Fatalf("write sources = %+v", loss.Record.WriteSources)
278 }
279 cons := loss.Record.ConfirmedConsequences
280 if !cons.RevokeExecution || !cons.RemoveParticipation || !cons.WritesPotentiallyLost || !cons.TimersPotentiallyLost || len(cons.DischargeLocal) != 0 || len(cons.DestroyedStorage) != 0 {
281 t.Fatalf("consequences = %+v", cons)
282 }
283 if b := w.roster(t).Roster.Boxes["uk-box"]; b.State != "quarantined" || !reflect.DeepEqual(b.LossIDs, []string{id}) {
284 t.Fatalf("roster = %+v", b)
285 }
286 c := w.claim(t)
287 if c.ETag != after.ETag {
288 t.Fatal("the answer is not the claim's last write")
289 }
290 op := c.Claim.Op
291 rb := op.Intent["rollback"]
292 if op.Kind != hostclaim.KindRestoring || op.Phase != hostclaim.PhaseReleased || op.Exec != "none" || !reflect.DeepEqual(op.Participants, []string{"ie-box"}) ||
293 !provision.ValidRollbackID(rb) || op.Journal["loss."+id] == "" || !hostclaim.LostTargetReleased(c.Claim, id) {
294 t.Fatalf("claim op = %+v", op)
295 }
296 if !reflect.DeepEqual(c.Claim.Lost, []string{"uk-box"}) || !reflect.DeepEqual(c.Claim.LossRefs, []string{id}) {
297 t.Fatalf("claim lost = %v refs = %v", c.Claim.Lost, c.Claim.LossRefs)
298 }
299 parks := map[string]string{}
300 for _, o := range c.Claim.Obligations {
301 if o.Kind == "park-local" {
302 t.Fatalf("a lost box was given a park-local: %+v", o)
303 }
304 parks[o.Region+"|"+o.Park.Location] = o.Owner
305 }
306 if !reflect.DeepEqual(parks, map[string]string{"uk|instances/" + lossHost: "region:uk", "uk|moves/" + lossHost + "/" + lossOp: "region:uk"}) {
307 t.Fatalf("parks = %v", parks)
308 }
309 rec, _ := w.record(t)
310 r := rec.Move.Rollback
311 if rec.Region != "ie" || rec.Move.State != provision.MoveRollingBack || r == nil || r.ID != rb || r.Reason != provision.RollbackDeclaredLost ||
312 r.TargetPhase != provision.RbTargetReleased || r.Dirty || r.LossRef != lossRef || r.Evidence != lossRef || r.SourcePhase != provision.RbSourceWaiting {
313 t.Fatalf("record move = %+v rollback = %+v", rec.Move, r)
314 }
315 // The source's waiting now takes the clean release, and its claimed CAS
316 // takes execution back on the declaration's authority.
317 if err := provision.MoveRollbackSourcePhase(provision.MoveGuard{ID: lossOp, Incarnation: w.inc}, provision.RbSourceWaiting, provision.RbSourceClaimed, nil)(&rec); err != nil {
318 t.Fatalf("source waiting: %v", err)
319 }
320 if err := hostclaim.ReturnToSource(lossOp, "ie-box", true, hostclaim.ReturnFence{})(c.Claim); err != nil {
321 t.Fatalf("source claimed: %v", err)
322 }
323 // A replay writes nothing more.
324 snap := lossSnapshot(t, w)
325 if _, err := w.declare(sub); err != nil {
326 t.Fatal(err)
327 }
328 if lossSnapshot(t, w) != snap {
329 t.Fatal("a replay wrote")
330 }
331}
332
333// The target released dirty and died before its return: the record goes
334// clean with the loss as evidence (no fake clean-return coverage: the
335// signed record names the target's sessions and the uncovered source cut),
336// and the possibly partial return lane is parked in the source region.
337func TestJ4DeclareLostDirtyReleasedTarget(t *testing.T) {
338 for _, state := range []string{"released-dirty", "returning"} {
339 t.Run(state, func(t *testing.T) {
340 w := newLossWorld(t, state, nil)
341 in := w.inspect(t, "uk-box")
342 if _, err := w.declare(w.submission(in)); err != nil {
343 t.Fatal(err)
344 }
345 loss, err := hostclaim.ReadLossRecord(context.Background(), w.raw, w.pub, in.Details.ID)
346 if err != nil || len(loss.Record.WriteSources) != 3 || loss.Record.ConfirmedConsequences.RevokeExecution {
347 t.Fatalf("loss = %+v %v", loss.Record, err)
348 }
349 c := w.claim(t)
350 parks := map[string]string{}
351 for _, o := range c.Claim.Obligations {
352 parks[o.Region+"|"+o.Park.Location] = o.ID
353 }
354 rb := "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb"
355 if parks["uk|instances/"+lossHost] != "loc" || parks["ie|moves/"+lossHost+"/"+rb] == "" || parks["uk|moves/"+lossHost+"/"+lossOp] == "" || len(parks) != 3 {
356 t.Fatalf("parks = %v", parks)
357 }
358 rec, _ := w.record(t)
359 if r := rec.Move.Rollback; r.TargetPhase != provision.RbTargetReleased || r.Dirty || r.Evidence == "" || r.Evidence != r.LossRef {
360 t.Fatalf("rollback = %+v", r)
361 }
362 })
363 }
364}
365
366func TestJ4DeclareLostReleasingTarget(t *testing.T) {
367 w := newLossWorld(t, "releasing", nil)
368 in := w.inspect(t, "uk-box")
369 if _, err := w.declare(w.submission(in)); err != nil {
370 t.Fatal(err)
371 }
372 c := w.claim(t)
373 if c.Claim.Op.Phase != hostclaim.PhaseReleased || c.Claim.Op.Intent["rollback"] != "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb" || c.Claim.Op.Exec != "none" {
374 t.Fatalf("op = %+v", c.Claim.Op)
375 }
376 rec, _ := w.record(t)
377 if r := rec.Move.Rollback; r.ID != "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb" || r.Reason != provision.RollbackProbeAttempts || r.TargetPhase != provision.RbTargetReleased || r.Dirty {
378 t.Fatalf("rollback = %+v", r)
379 }
380}
381
382// Every refusal comes before acceptance and quarantine.
383func TestJ4RefusalsBeforeAcceptance(t *testing.T) {
384 ctx := context.Background()
385 cases := map[string]struct {
386 state string
387 mutate func(*hostclaim.Claim, *provision.Record, *AppAdmission)
388 box string
389 }{
390 "source box": {"landing", nil, "ie-box"},
391 "no registration": {"landing", func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) { delete(a.Registrations, lossOp) }, "uk-box"},
392 "other registration": {"landing", func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) {
393 r := a.Registrations[lossOp]
394 r.Host = "other.example"
395 a.Registrations[lossOp] = r
396 }, "uk-box"},
397 "committed record": {"landing", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
398 r.Move.State = provision.MoveCommitted
399 at := time.Now()
400 r.Move.CommittedAt = &at
401 }, "uk-box"},
402 "committed claim": {"landing", func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) { c.Op.Phase = hostclaim.PhaseCommitted }, "uk-box"},
403 "moving claim (pre-flip)": {"landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
404 c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindMoving, hostclaim.PhaseSealed, ""
405 r.Move.State, r.Move.CutPhase, r.Move.FlippedAt, r.Region = provision.MoveCutting, provision.CutArchived, nil, "ie"
406 }, "uk-box"},
407 "stuck rollback": {"releasing", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
408 r.Move.State = provision.MoveStuck
409 }, "uk-box"},
410 "released clean": {"released-dirty", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
411 r.Move.Rollback.Dirty = false
412 }, "uk-box"},
413 "no seal": {"landing", func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) {
414 delete(c.Op.Journal, hostclaim.JournalSeal)
415 }, "uk-box"},
416 "other record move": {"landing", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
417 r.Move.ID = "mv_cccccccccccccccccccccccccc"
418 }, "uk-box"},
419 // G1: the claim still moving (sealed) while the record already landing.
420 "moving claim, landing record": {"landing", func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) {
421 c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindMoving, hostclaim.PhaseSealed, ""
422 }, "uk-box"},
423 }
424 for name, tc := range cases {
425 t.Run(name, func(t *testing.T) {
426 mutate := tc.mutate
427 w := newLossWorld(t, tc.state, mutate)
428 snap := lossSnapshot(t, w)
429 _, err := w.control.InspectMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, tc.box)
430 if err == nil {
431 t.Fatal("inspection accepted")
432 }
433 if lossSnapshot(t, w) != snap {
434 t.Fatal("a refused inspection wrote")
435 }
436 })
437 }
438 // Another app's door cannot inspect this host.
439 w0 := newLossWorld(t, "landing", nil)
440 if _, err := w0.control.InspectMoveLoss(ctx, w0.signer, lossBucket, "operator", "bab", "other", lossHost, "uk-box"); !errors.Is(err, ErrIdentity) {
441 t.Fatalf("other app = %v", err)
442 }
443 // The recorded refusal for a committed move reaches the door as its
444 // sentence.
445 w := newLossWorld(t, "landing", func(_ *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
446 r.Move.State = provision.MoveCommitted
447 at := time.Now()
448 r.Move.CommittedAt = &at
449 })
450 _, err := w.control.InspectMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box")
451 if !errors.Is(err, provision.ErrMoveConflict) || !strings.Contains(err.Error(), provision.RefusedCommitted) {
452 t.Fatalf("committed = %v", err)
453 }
454}
455
456// A declaration whose world changed after inspection is refused before
457// acceptance: a stale record, a stale claim, a removed registration.
458func TestJ4StaleObservationsRefusedBeforeAcceptance(t *testing.T) {
459 ctx := context.Background()
460 for name, change := range map[string]func(*testing.T, *lossWorld){
461 "record": func(t *testing.T, w *lossWorld) {
462 if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw.Store, "bab", "app", lossHost, func(r *provision.Record) error {
463 r.Move.LandingPhase = provision.LandProbed
464 return nil
465 }); err != nil {
466 t.Fatal(err)
467 }
468 },
469 "claim": func(t *testing.T, w *lossWorld) {
470 if _, err := hostclaim.Update(ctx, w.raw.Store, w.claim(t), func(*hostclaim.Claim) error { return nil }); err != nil {
471 t.Fatal(err)
472 }
473 },
474 "registration": func(t *testing.T, w *lossWorld) {
475 a, err := ReadAdmission(ctx, w.raw, "bab", "app", "canonical")
476 if err != nil {
477 t.Fatal(err)
478 }
479 delete(a.Value.Registrations, lossOp)
480 b, _ := json.Marshal(a.Value)
481 if _, err := w.raw.Store.Put(ctx, a.Key, b, a.ETag); err != nil {
482 t.Fatal(err)
483 }
484 },
485 } {
486 t.Run(name, func(t *testing.T) {
487 w := newLossWorld(t, "landing", nil)
488 sub := w.submission(w.inspect(t, "uk-box"))
489 change(t, w)
490 snap := lossSnapshot(t, w)
491 if _, err := w.declare(sub); err == nil {
492 t.Fatal("stale declaration accepted")
493 }
494 if lossSnapshot(t, w) != snap {
495 t.Fatal("a refused declaration wrote")
496 }
497 })
498 }
499}
500
501// The submission is the signed challenge: another host, another box,
502// another account or app, or changed bytes are refused before acceptance.
503func TestJ4ChangedRequestRefused(t *testing.T) {
504 ctx := context.Background()
505 w := newLossWorld(t, "landing", nil)
506 sub := w.submission(w.inspect(t, "uk-box"))
507 snap := lossSnapshot(t, w)
508 for name, call := range map[string]func() error{
509 "other host": func() error {
510 _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", "other.example", "uk-box", sub)
511 return err
512 },
513 "other box": func() error {
514 _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "ie-box", sub)
515 return err
516 },
517 "other app": func() error {
518 _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "other", lossHost, "uk-box", sub)
519 return err
520 },
521 "other actor": func() error {
522 _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "someone", "bab", "app", lossHost, "uk-box", sub)
523 return err
524 },
525 "changed body": func() error {
526 bad := sub
527 bad.Challenge = append([]byte(nil), sub.Challenge...)
528 bad.Challenge[len(bad.Challenge)/2] ^= 1
529 _, err := w.control.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", bad)
530 return err
531 },
532 "other signer": func() error {
533 _, priv, _ := ed25519.GenerateKey(rand.Reader)
534 _, err := w.control.DeclareMoveLoss(ctx, release.RootSigner(priv), lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub)
535 return err
536 },
537 "other bucket": func() error {
538 _, err := w.control.DeclareMoveLoss(ctx, w.signer, "other-bucket", "operator", "bab", "app", lossHost, "uk-box", sub)
539 return err
540 },
541 } {
542 if err := call(); err == nil {
543 t.Fatalf("%s accepted", name)
544 }
545 if lossSnapshot(t, w) != snap {
546 t.Fatalf("%s wrote", name)
547 }
548 }
549}
550
551// Expiry and closed admission do not revoke an accepted move.
552func TestJ4ExpiredRegistrationAndClosedAdmissionStillDeclare(t *testing.T) {
553 w := newLossWorld(t, "landing", func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) {
554 r := a.Registrations[lossOp]
555 r.Expires = time.Now().Add(-48 * time.Hour)
556 a.Registrations[lossOp] = r
557 a.State = "closed"
558 })
559 if _, err := w.declare(w.submission(w.inspect(t, "uk-box"))); err != nil {
560 t.Fatal(err)
561 }
562}
563
564// Crash and lost-answer points across the durable sequence: each fault
565// leaves a state the same submission completes, with nothing written twice.
566func TestJ4FaultsResumeFromDurableState(t *testing.T) {
567 for _, fault := range []struct {
568 name, key string
569 before, lost bool
570 nth int
571 }{
572 {"record prepare fails", "record", true, false, 0},
573 {"record prepare answer lost", "record", false, true, 0},
574 {"declare CAS fails", "claim", true, false, 0},
575 {"declare CAS answer lost", "claim", false, true, 0},
576 {"loss commit fails", "loss", false, false, 2},
577 {"release CAS fails", "claim", false, false, 2},
578 {"loss-committed record write fails", "record", false, false, 2},
579 } {
580 t.Run(fault.name, func(t *testing.T) {
581 w := newLossWorld(t, "landing", nil)
582 sub := w.submission(w.inspect(t, "uk-box"))
583 key := map[string]string{"record": w.key, "claim": w.claimK}[fault.key]
584 if fault.key == "loss" {
585 key, _ = hostclaim.LossRecordKey(sub.Confirmation)
586 }
587 var err error
588 if fault.nth > 0 {
589 hook := &lossNthPut{lossFaultStore: w.raw, key: key, n: fault.nth}
590 cw, cerr := NewControlWriter(hook, w.pub, func() time.Time { return w.clock })
591 if cerr != nil {
592 t.Fatal(cerr)
593 }
594 _, err = cw.DeclareMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub)
595 } else {
596 w.raw.key, w.raw.failBefore, w.raw.failAfter = key, fault.before, fault.lost
597 _, err = w.declare(sub)
598 }
599 if err == nil {
600 t.Fatal("fault not reached")
601 }
602 if _, err := w.declare(sub); err != nil {
603 t.Fatalf("resume: %v", err)
604 }
605 c := w.claim(t)
606 rec, _ := w.record(t)
607 if c.Claim.Op.Phase != hostclaim.PhaseReleased || rec.Move.Rollback.TargetPhase != provision.RbTargetReleased || len(c.Claim.Obligations) != 2 {
608 t.Fatalf("after resume: op %+v rollback %+v obligations %d", c.Claim.Op, rec.Move.Rollback, len(c.Claim.Obligations))
609 }
610 snap := lossSnapshot(t, w)
611 if _, err := w.declare(sub); err != nil || lossSnapshot(t, w) != snap {
612 t.Fatalf("second replay: %v (wrote %v)", err, lossSnapshot(t, w) != snap)
613 }
614 })
615 }
616}
617
618// lossNthPut fails the nth Put of key (before it lands).
619type lossNthPut struct {
620 *lossFaultStore
621 key string
622 n int
623}
624
625func (s *lossNthPut) Put(ctx context.Context, key string, body []byte, match string) (string, error) {
626 if key == s.key {
627 s.n--
628 if s.n == 0 {
629 return "", errors.New("injected put failure")
630 }
631 }
632 return s.lossFaultStore.Store.Put(ctx, key, body, match)
633}
634
635// The owner's hooks are idempotent and never regress: a late PrepareLoss
636// after FinishLoss writes nothing; a FinishLoss replay writes nothing; a
637// PrepareLoss for a conflicted declaration writes nothing; a FinishLoss
638// facing a successor holder with no durable completion refuses.
639func TestJ4OwnerHooksLateAndConflicted(t *testing.T) {
640 ctx := context.Background()
641 w := newLossWorld(t, "landing", nil)
642 sub := w.submission(w.inspect(t, "uk-box"))
643 if _, err := w.declare(sub); err != nil {
644 t.Fatal(err)
645 }
646 loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub.Confirmation)
647 if err != nil {
648 t.Fatal(err)
649 }
650 owner := &moveLossOwner{store: w.raw, pub: w.pub}
651 snap := lossSnapshot(t, w)
652 if err := owner.PrepareLoss(ctx, w.claim(t), loss.Record); err != nil {
653 t.Fatalf("late prepare: %v", err)
654 }
655 if err := owner.FinishLoss(ctx, w.claim(t), loss.Record); err != nil {
656 t.Fatalf("finish replay: %v", err)
657 }
658 if lossSnapshot(t, w) != snap {
659 t.Fatal("a late hook wrote")
660 }
661
662 // A conflicted declaration: its prepare writes nothing.
663 w2 := newLossWorld(t, "landing", nil)
664 sub2 := w2.submission(w2.inspect(t, "uk-box"))
665 // Accept and prepare by failing the record write once, then mark the
666 // declaration conflicted by moving the claim.
667 w2.raw.key, w2.raw.failBefore = w2.key, true
668 if _, err := w2.declare(sub2); err == nil {
669 t.Fatal("fault not reached")
670 }
671 if _, err := hostclaim.Update(ctx, w2.raw.Store, w2.claim(t), func(*hostclaim.Claim) error { return nil }); err != nil {
672 t.Fatal(err)
673 }
674 if _, err := w2.declare(sub2); !errors.Is(err, hostclaim.ErrConflict) {
675 t.Fatalf("conflicted retry = %v", err)
676 }
677 conflicted, err := hostclaim.ReadLossRecord(ctx, w2.raw, w2.pub, sub2.Confirmation)
678 if err != nil || conflicted.Record.State != "conflicted" {
679 t.Fatalf("loss = %+v %v", conflicted.Record, err)
680 }
681 snap2 := lossSnapshot(t, w2)
682 prepared := conflicted.Record
683 prepared.State = "prepared"
684 if err := (&moveLossOwner{store: w2.raw, pub: w2.pub}).PrepareLoss(ctx, w2.claim(t), prepared); err == nil {
685 t.Fatal("prepare of a conflicted declaration accepted")
686 }
687 if lossSnapshot(t, w2) != snap2 {
688 t.Fatal("prepare of a conflicted declaration wrote")
689 }
690 if rec, _ := w2.record(t); rec.Move.State != provision.MoveLanding {
691 t.Fatalf("record = %+v", rec.Move)
692 }
693
694 // A successor holder before the release: FinishLoss refuses and writes
695 // nothing.
696 w3 := newLossWorld(t, "landing", nil)
697 sub3 := w3.submission(w3.inspect(t, "uk-box"))
698 hook := &lossNthPut{lossFaultStore: w3.raw, key: w3.claimK, n: 2}
699 cw, err := NewControlWriter(hook, w3.pub, func() time.Time { return w3.clock })
700 if err != nil {
701 t.Fatal(err)
702 }
703 if _, err := cw.DeclareMoveLoss(ctx, w3.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub3); err == nil {
704 t.Fatal("release fault not reached")
705 }
706 loss3, err := hostclaim.ReadLossRecord(ctx, w3.raw, w3.pub, sub3.Confirmation)
707 if err != nil || loss3.Record.State != "committed" {
708 t.Fatalf("loss = %+v %v", loss3.Record, err)
709 }
710 if _, err := hostclaim.Update(ctx, w3.raw.Store, w3.claim(t), func(c *hostclaim.Claim) error {
711 c.Holder.CreatedAt = c.Holder.CreatedAt.Add(time.Second)
712 inc, err := hostclaim.Incarnation("bab", "app", lossHost, c.Holder.CreatedAt)
713 c.Holder.Incarnation = inc
714 return err
715 }); err != nil {
716 t.Fatal(err)
717 }
718 snap3 := lossSnapshot(t, w3)
719 if err := (&moveLossOwner{store: w3.raw, pub: w3.pub}).FinishLoss(ctx, w3.claim(t), loss3.Record); err == nil {
720 t.Fatal("finish against a successor holder accepted")
721 }
722 if lossSnapshot(t, w3) != snap3 {
723 t.Fatal("finish against a successor wrote")
724 }
725}
726
727// ValidateDeclareLost is pure and exact: changed signed effects are refused
728// inside the claim CAS.
729func TestJ4ValidateDeclareLostRefusesChangedEffects(t *testing.T) {
730 ctx := context.Background()
731 w := newLossWorld(t, "landing", nil)
732 o := w.claim(t)
733 owner := &moveLossOwner{store: w.raw, pub: w.pub}
734 _, tag := w.record(t)
735 _, actual, err := owner.InspectLoss(ctx, o, hostfence.RecordObservation{Key: w.key, ETag: tag, CommitState: "present"}, "uk-box")
736 if err != nil {
737 t.Fatal(err)
738 }
739 d := hostclaim.LostDeclaration{LossID: "loss_1", ExpectedClaimGen: o.Claim.Gen, ExpectedClaimETag: o.ETag, HolderIncarnation: w.inc, HolderStamp: o.Claim.Holder.Stamp,
740 HolderSession: o.Claim.Holder.Session, OpID: lossOp, OpKind: hostclaim.KindLanding, OpPhase: "probing", Box: "uk-box", QuarantineRevision: 1, Enrollment: strings.Repeat("b", 32), FencedAt: w.clock, Consequences: actual}
741 if err := owner.ValidateDeclareLost(*o.Claim, d); err != nil {
742 t.Fatal(err)
743 }
744 for name, change := range map[string]func(*hostclaim.LostDeclaration){
745 "keep execution": func(d *hostclaim.LostDeclaration) { d.Consequences.RevokeExecution = false },
746 "no writes lost": func(d *hostclaim.LostDeclaration) { d.Consequences.WritesPotentiallyLost = false },
747 "destroyed storage": func(d *hostclaim.LostDeclaration) { d.Consequences.DestroyedStorage = []string{"x"} },
748 "other box": func(d *hostclaim.LostDeclaration) { d.Box = "ie-box" },
749 "other etag": func(d *hostclaim.LostDeclaration) { d.ExpectedClaimETag = "\"other\"" },
750 "intent": func(d *hostclaim.LostDeclaration) {
751 d.Consequences.RecoveryIntent = json.RawMessage(`{"kind":"move-target-lost"}`)
752 },
753 "intent move": func(d *hostclaim.LostDeclaration) {
754 var m map[string]any
755 _ = json.Unmarshal(d.Consequences.RecoveryIntent, &m)
756 m["move"] = "mv_cccccccccccccccccccccccccc"
757 d.Consequences.RecoveryIntent, _ = json.Marshal(m)
758 },
759 "intent target sessions": func(d *hostclaim.LostDeclaration) {
760 var m map[string]any
761 _ = json.Unmarshal(d.Consequences.RecoveryIntent, &m)
762 m["src_covered"] = true
763 d.Consequences.RecoveryIntent, _ = json.Marshal(m)
764 },
765 } {
766 changed := d
767 changed.Consequences.RecoveryIntent = append(json.RawMessage(nil), d.Consequences.RecoveryIntent...)
768 change(&changed)
769 if err := owner.ValidateDeclareLost(*o.Claim, changed); err == nil {
770 t.Errorf("%s accepted", name)
771 }
772 }
773}
774
775// The scoped store behind the declaration admits only the bound
776// declaration's objects and exact transitions; a raw write, a delete, an
777// unrelated key or a different transition through it is refused.
778func TestJ4ScopedStoreRefusesEverythingElse(t *testing.T) {
779 ctx := context.Background()
780 w := newLossWorld(t, "landing", nil)
781 o := w.claim(t)
782 s := &lossScopedStore{raw: w.raw, pub: w.pub, now: func() time.Time { return w.clock },
783 b: lossBinding{lossID: "loss_1", host: lossHost, opID: lossOp, kind: hostclaim.KindLanding, box: "uk-box", claimETag: o.ETag, incarnation: w.inc, account: "bab", app: "app"}}
784 snap := lossSnapshot(t, w)
785 if _, err := s.Put(ctx, "apps/bab/app/other.json", []byte(`{}`), ""); !errors.Is(err, ErrContract) {
786 t.Fatalf("unrelated key = %v", err)
787 }
788 if err := s.Delete(ctx, w.key); !errors.Is(err, ErrContract) {
789 t.Fatalf("delete = %v", err)
790 }
791 // A loss record or acceptance for another declaration id.
792 if _, err := s.Put(ctx, "control/host-loss/loss_2.json", []byte(`{}`), ""); !errors.Is(err, ErrContract) {
793 t.Fatalf("other loss = %v", err)
794 }
795 if _, err := s.Put(ctx, "control/host-loss-acceptances/loss_1.json", []byte(`{}`), ""); err == nil {
796 t.Fatal("unsigned acceptance accepted")
797 }
798 // The roster, unsigned or with another change.
799 if _, err := s.Put(ctx, hostactors.Key, []byte(`{}`), w.roster(t).ETag); err == nil {
800 t.Fatal("unsigned roster accepted")
801 }
802 ro := w.roster(t)
803 r := hostactors.Clone(ro.Roster)
804 b := r.Boxes["ie-box"]
805 b.State, b.QuarantineRevision, b.LossIDs = "quarantined", 1, []string{"loss_1"}
806 r.Boxes["ie-box"] = b
807 if _, err := (hostactors.Directory{Store: s, PublicKey: w.pub, Bucket: lossBucket}).Replace(ctx, w.signer, ro, r); err == nil {
808 t.Fatal("quarantine of another box accepted")
809 }
810 if lossSnapshot(t, w) != snap {
811 t.Fatal("a refused scoped write landed")
812 }
813}
814
815// Only the move kinds are registered, and only on the typed methods: a
816// serving claim is not resolvable through them.
817func TestJ4OnlyMoveKindsAreServed(t *testing.T) {
818 w := newLossWorld(t, "landing", func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
819 c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "ie-box"}
820 r.Move = nil
821 })
822 if _, err := w.control.InspectMoveLoss(context.Background(), w.signer, lossBucket, "operator", "bab", "app", lossHost, "ie-box"); !errors.Is(err, hostresolve.ErrResolve) {
823 t.Fatalf("serving = %v", err)
824 }
825}
826
827// lossAfterPut runs after once, right after the first Put of key lands.
828type lossAfterPut struct {
829 *lossFaultStore
830 key string
831 after func()
832}
833
834func (s *lossAfterPut) Put(ctx context.Context, key string, body []byte, match string) (string, error) {
835 tag, err := s.lossFaultStore.Store.Put(ctx, key, body, match)
836 if err == nil && key == s.key && s.after != nil {
837 f := s.after
838 s.after = nil
839 f()
840 }
841 return tag, err
842}
843
844// The registration is reread at every scoped write, not only at entry: one
845// removed after the acceptance stops the quarantine and everything after it.
846func TestJ4RegistrationRecheckedAtEachWrite(t *testing.T) {
847 ctx := context.Background()
848 w := newLossWorld(t, "landing", nil)
849 sub := w.submission(w.inspect(t, "uk-box"))
850 roster := w.roster(t).ETag
851 hook := &lossAfterPut{lossFaultStore: w.raw, key: "control/host-loss-acceptances/" + sub.Confirmation + ".json", after: func() {
852 a, err := ReadAdmission(ctx, w.raw, "bab", "app", "canonical")
853 if err != nil {
854 t.Fatal(err)
855 }
856 delete(a.Value.Registrations, lossOp)
857 b, _ := json.Marshal(a.Value)
858 if _, err := w.raw.Store.Put(ctx, a.Key, b, a.ETag); err != nil {
859 t.Fatal(err)
860 }
861 }}
862 cw, err := NewControlWriter(hook, w.pub, func() time.Time { return w.clock })
863 if err != nil {
864 t.Fatal(err)
865 }
866 before := w.claim(t)
867 rec, tag := w.record(t)
868 if _, err := cw.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub); !errors.Is(err, ErrPending) {
869 t.Fatalf("declaration after the registration went = %v", err)
870 }
871 if w.roster(t).ETag != roster {
872 t.Fatal("quarantined without the registration")
873 }
874 if c := w.claim(t); c.ETag != before.ETag {
875 t.Fatal("claim written without the registration")
876 }
877 if after, tag2 := w.record(t); tag2 != tag || after.Move.State != rec.Move.State {
878 t.Fatal("record written without the registration")
879 }
880}
881
882// A record changed after acceptance (a late box write) is not overwritten
883// by the declaration's decision: the prepare refuses and the declaration
884// stays prepared, with the claim untouched.
885func TestJ4RecordChangedAfterAcceptanceRefused(t *testing.T) {
886 ctx := context.Background()
887 w := newLossWorld(t, "landing", nil)
888 sub := w.submission(w.inspect(t, "uk-box"))
889 lossKey, _ := hostclaim.LossRecordKey(sub.Confirmation)
890 hook := &lossAfterPut{lossFaultStore: w.raw, key: lossKey, after: func() {
891 if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw.Store, "bab", "app", lossHost, func(r *provision.Record) error {
892 r.Move.LandingPhase = provision.LandProbed
893 return nil
894 }); err != nil {
895 t.Fatal(err)
896 }
897 }}
898 cw, err := NewControlWriter(hook, w.pub, func() time.Time { return w.clock })
899 if err != nil {
900 t.Fatal(err)
901 }
902 before := w.claim(t)
903 if _, err := cw.DeclareMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box", sub); !errors.Is(err, hostclaim.ErrConflict) {
904 t.Fatalf("declaration over a changed record = %v", err)
905 }
906 rec, _ := w.record(t)
907 if rec.Move.State != provision.MoveLanding || rec.Move.LandingPhase != provision.LandProbed || rec.Move.Rollback != nil {
908 t.Fatalf("record = %+v", rec.Move)
909 }
910 if c := w.claim(t); c.ETag != before.ETag {
911 t.Fatal("claim written over a changed record")
912 }
913 loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub.Confirmation)
914 if err != nil || loss.Record.State != "prepared" {
915 t.Fatalf("loss = %+v %v", loss.Record, err)
916 }
917}
918
919// conflictAfterPrepare drives a declaration to B1's stranded state: the
920// record prepare lands (lost with this loss as its reference), the claim
921// CAS fails, an unrelated claim write moves the ETag, and the retry marks
922// the declaration conflicted.
923func conflictAfterPrepare(t *testing.T, w *lossWorld, sub hostfence.Submission) {
924 t.Helper()
925 ctx := context.Background()
926 w.raw.key, w.raw.failBefore = w.claimK, true
927 if _, err := w.declare(sub); err == nil {
928 t.Fatal("claim fault not reached")
929 }
930 ref, _ := hostclaim.LossRecordKey(sub.Confirmation)
931 if rec, _ := w.record(t); rec.Move.Rollback == nil || rec.Move.Rollback.TargetPhase != provision.RbTargetLost || rec.Move.Rollback.LossRef != ref {
932 t.Fatalf("record after prepare = %+v", rec.Move)
933 }
934 if _, err := hostclaim.Update(ctx, w.raw.Store, w.claim(t), func(*hostclaim.Claim) error { return nil }); err != nil {
935 t.Fatal(err)
936 }
937 if _, err := w.declare(sub); !errors.Is(err, hostclaim.ErrConflict) {
938 t.Fatalf("conflicted retry = %v", err)
939 }
940 if loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub.Confirmation); err != nil || loss.Record.State != "conflicted" {
941 t.Fatalf("loss = %+v %v", loss.Record, err)
942 }
943}
944
945// B1: a declaration conflicted after its record prepare leaves the record
946// lost with a dead reference. A fresh declaration rebinds it and completes,
947// keeping the rollback the first one decided, and the source takes over.
948func TestJ4ConflictedPreparedLossRebinds(t *testing.T) {
949 ctx := context.Background()
950 w := newLossWorld(t, "landing", nil)
951 sub1 := w.submission(w.inspect(t, "uk-box"))
952 conflictAfterPrepare(t, w, sub1)
953 rec1, _ := w.record(t)
954 rb1 := rec1.Move.Rollback.ID
955
956 in2, err := w.control.InspectMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box")
957 if err != nil {
958 t.Fatalf("fresh inspection: %v", err)
959 }
960 sub2 := w.submission(in2)
961 if _, err := w.declare(sub2); err != nil {
962 t.Fatalf("fresh declaration: %v", err)
963 }
964 ref2, _ := hostclaim.LossRecordKey(sub2.Confirmation)
965 c := w.claim(t)
966 if c.Claim.Op.Kind != hostclaim.KindRestoring || c.Claim.Op.Phase != hostclaim.PhaseReleased || c.Claim.Op.Intent["rollback"] != rb1 ||
967 !hostclaim.LostTargetReleased(c.Claim, sub2.Confirmation) || !reflect.DeepEqual(c.Claim.LossRefs, []string{sub2.Confirmation}) {
968 t.Fatalf("claim = %+v refs %v", c.Claim.Op, c.Claim.LossRefs)
969 }
970 rec, _ := w.record(t)
971 if r := rec.Move.Rollback; r.ID != rb1 || r.TargetPhase != provision.RbTargetReleased || r.Dirty || r.LossRef != ref2 || r.Evidence != ref2 || r.Reason != provision.RollbackDeclaredLost {
972 t.Fatalf("rollback = %+v", r)
973 }
974 if err := provision.MoveRollbackSourcePhase(provision.MoveGuard{ID: lossOp, Incarnation: w.inc}, provision.RbSourceWaiting, provision.RbSourceClaimed, nil)(&rec); err != nil {
975 t.Fatalf("source waiting: %v", err)
976 }
977 if err := hostclaim.ReturnToSource(lossOp, "ie-box", true, hostclaim.ReturnFence{})(c.Claim); err != nil {
978 t.Fatalf("source claimed: %v", err)
979 }
980}
981
982// lossIntentFor reads a declaration's signed intent.
983func lossIntentFor(t *testing.T, w *lossWorld, id string) moveLossIntent {
984 t.Helper()
985 loss, err := hostclaim.ReadLossRecord(context.Background(), w.raw, w.pub, id)
986 if err != nil {
987 t.Fatal(err)
988 }
989 in, err := decodeMoveLossIntent(loss.Record.ConfirmedConsequences.RecoveryIntent)
990 if err != nil {
991 t.Fatal(err)
992 }
993 return in
994}
995
996// forgeLoss rewrites a signed loss record with the root signer (standing in
997// for any signed state the test needs) and returns the bytes it replaced.
998func forgeLoss(t *testing.T, w *lossWorld, id string, change func(*hostclaim.LossRecord)) []byte {
999 t.Helper()
1000 ctx := context.Background()
1001 key, _ := hostclaim.LossRecordKey(id)
1002 old, tag, err := w.raw.Store.Get(ctx, key)
1003 if err != nil {
1004 t.Fatal(err)
1005 }
1006 loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, id)
1007 if err != nil {
1008 t.Fatal(err)
1009 }
1010 r := loss.Record
1011 change(&r)
1012 wire, err := release.SignRootEnvelope(w.signer, r)
1013 if err != nil {
1014 t.Fatal(err)
1015 }
1016 if _, err := w.raw.Store.Put(ctx, key, wire, tag); err != nil {
1017 t.Fatal(err)
1018 }
1019 return old
1020}
1021
1022// A dirty target's rebind keeps the signed dirty phase and its possibly
1023// partial return lane, across a chain of conflicted declarations; the
1024// rollback the first declaration found stands throughout, and a late
1025// prepare of a superseded declaration writes nothing.
1026func TestJ4ConflictedLossChainKeepsSignedPhase(t *testing.T) {
1027 ctx := context.Background()
1028 for _, tc := range []struct {
1029 state, phase string
1030 dirty, lane bool
1031 }{
1032 {"landing", provision.RbTargetReleasing, false, false},
1033 {"released-dirty", provision.RbTargetReleased, true, true},
1034 {"returning", provision.RbTargetReturning, true, true},
1035 } {
1036 t.Run(tc.state, func(t *testing.T) {
1037 w := newLossWorld(t, tc.state, nil)
1038 sub1 := w.submission(w.inspect(t, "uk-box"))
1039 conflictAfterPrepare(t, w, sub1)
1040 rec1, _ := w.record(t)
1041 rb1 := rec1.Move.Rollback.ID
1042 ref1, _ := hostclaim.LossRecordKey(sub1.Confirmation)
1043 sub2 := w.submission(w.inspect(t, "uk-box"))
1044 conflictAfterPrepare(t, w, sub2)
1045 ref2, _ := hostclaim.LossRecordKey(sub2.Confirmation)
1046 sub3 := w.submission(w.inspect(t, "uk-box"))
1047 if _, err := w.declare(sub3); err != nil {
1048 t.Fatalf("third declaration: %v", err)
1049 }
1050 for id, supersedes := range map[string]string{sub2.Confirmation: ref1, sub3.Confirmation: ref2} {
1051 in := lossIntentFor(t, w, id)
1052 if in.Supersedes != supersedes || in.Rollback != rb1 || in.RecordState != moveLossRecordRollingBack || in.TargetPhase != tc.phase || in.Dirty != tc.dirty || in.ReturnLane != tc.lane {
1053 t.Fatalf("intent %s = %+v", id, in)
1054 }
1055 }
1056 ref3, _ := hostclaim.LossRecordKey(sub3.Confirmation)
1057 rec, _ := w.record(t)
1058 if r := rec.Move.Rollback; r.ID != rb1 || r.TargetPhase != provision.RbTargetReleased || r.Dirty || r.Evidence != ref3 || r.LossRef != ref3 {
1059 t.Fatalf("rollback = %+v", r)
1060 }
1061 c := w.claim(t)
1062 if c.Claim.Op.Intent["rollback"] != rb1 || !reflect.DeepEqual(c.Claim.LossRefs, []string{sub3.Confirmation}) {
1063 t.Fatalf("claim = %+v %v", c.Claim.Op, c.Claim.LossRefs)
1064 }
1065 lane := false
1066 for _, o := range c.Claim.Obligations {
1067 lane = lane || (o.Region == "ie" && o.Park.Location == "moves/"+lossHost+"/"+rb1)
1068 }
1069 if lane != tc.lane {
1070 t.Fatalf("return lane parked = %v, obligations %+v", lane, c.Claim.Obligations)
1071 }
1072 // Late hooks of the superseded declarations write nothing.
1073 snap := lossSnapshot(t, w)
1074 owner := &moveLossOwner{store: w.raw, pub: w.pub}
1075 for _, id := range []string{sub1.Confirmation, sub2.Confirmation} {
1076 loss, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, id)
1077 if err != nil {
1078 t.Fatal(err)
1079 }
1080 prepared := loss.Record
1081 prepared.State = "prepared"
1082 _ = owner.PrepareLoss(ctx, c, prepared)
1083 if _, err := w.declare(map[string]hostfence.Submission{sub1.Confirmation: sub1, sub2.Confirmation: sub2}[id]); !errors.Is(err, hostclaim.ErrConflict) {
1084 t.Fatalf("superseded replay = %v", err)
1085 }
1086 }
1087 if lossSnapshot(t, w) != snap {
1088 t.Fatal("a superseded declaration wrote")
1089 }
1090 })
1091 }
1092}
1093
1094// Only a signed, conflicted declaration of this move's target whose rollback
1095// and dirty bit the record still carries may be superseded; everything
1096// else stays refused before acceptance.
1097func TestJ4ConflictedLossRebindRefusals(t *testing.T) {
1098 ctx := context.Background()
1099 for name, setup := range map[string]func(*testing.T, *lossWorld){
1100 "old prepared": func(t *testing.T, w *lossWorld) {
1101 sub := w.submission(w.inspect(t, "uk-box"))
1102 w.raw.key, w.raw.failBefore = w.claimK, true
1103 if _, err := w.declare(sub); err == nil {
1104 t.Fatal("claim fault not reached")
1105 }
1106 },
1107 "old committed": func(t *testing.T, w *lossWorld) {
1108 sub := w.submission(w.inspect(t, "uk-box"))
1109 conflictAfterPrepare(t, w, sub)
1110 forgeLoss(t, w, sub.Confirmation, func(r *hostclaim.LossRecord) { r.State, r.ResultClaimGen = "committed", r.ExpectedClaimGen+1 })
1111 },
1112 "old other host": func(t *testing.T, w *lossWorld) {
1113 sub := w.submission(w.inspect(t, "uk-box"))
1114 conflictAfterPrepare(t, w, sub)
1115 forgeLoss(t, w, sub.Confirmation, func(r *hostclaim.LossRecord) { r.Host = "other.example" })
1116 },
1117 "old other rollback": func(t *testing.T, w *lossWorld) {
1118 sub := w.submission(w.inspect(t, "uk-box"))
1119 conflictAfterPrepare(t, w, sub)
1120 forgeLoss(t, w, sub.Confirmation, func(r *hostclaim.LossRecord) {
1121 var m map[string]any
1122 _ = json.Unmarshal(r.ConfirmedConsequences.RecoveryIntent, &m)
1123 m["rollback"] = "rb_cccccccccccccccccccccccccc"
1124 r.ConfirmedConsequences.RecoveryIntent, _ = json.Marshal(m)
1125 })
1126 },
1127 "old other incarnation": func(t *testing.T, w *lossWorld) {
1128 sub := w.submission(w.inspect(t, "uk-box"))
1129 conflictAfterPrepare(t, w, sub)
1130 forgeLoss(t, w, sub.Confirmation, func(r *hostclaim.LossRecord) {
1131 var m map[string]any
1132 _ = json.Unmarshal(r.ConfirmedConsequences.RecoveryIntent, &m)
1133 m["incarnation"] = "other"
1134 r.ConfirmedConsequences.RecoveryIntent, _ = json.Marshal(m)
1135 r.HolderIncarnation = "other"
1136 })
1137 },
1138 "old unsigned": func(t *testing.T, w *lossWorld) {
1139 sub := w.submission(w.inspect(t, "uk-box"))
1140 conflictAfterPrepare(t, w, sub)
1141 key, _ := hostclaim.LossRecordKey(sub.Confirmation)
1142 _, tag, _ := w.raw.Store.Get(ctx, key)
1143 _, priv, _ := ed25519.GenerateKey(rand.Reader)
1144 loss, _ := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub.Confirmation)
1145 wire, _ := release.SignRootEnvelope(release.RootSigner(priv), loss.Record)
1146 if _, err := w.raw.Store.Put(ctx, key, wire, tag); err != nil {
1147 t.Fatal(err)
1148 }
1149 },
1150 "record names a fence": func(t *testing.T, w *lossWorld) {
1151 sub := w.submission(w.inspect(t, "uk-box"))
1152 conflictAfterPrepare(t, w, sub)
1153 if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw.Store, "bab", "app", lossHost, func(r *provision.Record) error {
1154 r.Move.Rollback.LossRef = "control/host-fences/f.json"
1155 return nil
1156 }); err != nil {
1157 t.Fatal(err)
1158 }
1159 },
1160 "record dirty changed": func(t *testing.T, w *lossWorld) {
1161 sub := w.submission(w.inspect(t, "uk-box"))
1162 conflictAfterPrepare(t, w, sub)
1163 if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw.Store, "bab", "app", lossHost, func(r *provision.Record) error {
1164 r.Move.Rollback.Dirty = true
1165 return nil
1166 }); err != nil {
1167 t.Fatal(err)
1168 }
1169 },
1170 } {
1171 t.Run(name, func(t *testing.T) {
1172 w := newLossWorld(t, "landing", nil)
1173 setup(t, w)
1174 snap := lossSnapshot(t, w)
1175 if _, err := w.control.InspectMoveLoss(ctx, w.signer, lossBucket, "operator", "bab", "app", lossHost, "uk-box"); err == nil {
1176 t.Fatal("inspection accepted")
1177 }
1178 if lossSnapshot(t, w) != snap {
1179 t.Fatal("a refused inspection wrote")
1180 }
1181 })
1182 }
1183}
1184
1185// The rebind's record write rereads the superseded declaration at the
1186// final write: a superseded declaration no longer signed conflicted stops
1187// both the owner and the scoped store; restored, the same declaration
1188// completes.
1189func TestJ4ConflictedLossRebindRecheckedAtWrite(t *testing.T) {
1190 ctx := context.Background()
1191 w := newLossWorld(t, "landing", nil)
1192 sub1 := w.submission(w.inspect(t, "uk-box"))
1193 conflictAfterPrepare(t, w, sub1)
1194 sub2 := w.submission(w.inspect(t, "uk-box"))
1195 w.raw.key, w.raw.failBefore = w.key, true
1196 if _, err := w.declare(sub2); err == nil {
1197 t.Fatal("record fault not reached")
1198 }
1199 loss2, err := hostclaim.ReadLossRecord(ctx, w.raw, w.pub, sub2.Confirmation)
1200 if err != nil || loss2.Record.State != "prepared" {
1201 t.Fatalf("loss = %+v %v", loss2.Record, err)
1202 }
1203 conflicted := forgeLoss(t, w, sub1.Confirmation, func(r *hostclaim.LossRecord) { r.State = "prepared" })
1204 snap := lossSnapshot(t, w)
1205 if err := (&moveLossOwner{store: w.raw, pub: w.pub}).PrepareLoss(ctx, w.claim(t), loss2.Record); err == nil {
1206 t.Fatal("owner rebind over a prepared declaration accepted")
1207 }
1208 in2, ref2, err := lossIntentOf(loss2.Record)
1209 if err != nil {
1210 t.Fatal(err)
1211 }
1212 s := &lossScopedStore{raw: w.raw, pub: w.pub, now: func() time.Time { return w.clock },
1213 b: lossBinding{lossID: sub2.Confirmation, host: lossHost, opID: lossOp, kind: hostclaim.KindLanding, box: "uk-box", claimETag: loss2.Record.ExpectedClaimETag, incarnation: w.inc, account: "bab", app: "app"}}
1214 if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), s, "bab", "app", lossHost, func(r *provision.Record) error {
1215 return prepareMoveLoss(r, in2, ref2, loss2.Record.FencedAt)
1216 }); err == nil {
1217 t.Fatal("scoped rebind over a prepared declaration accepted")
1218 }
1219 if lossSnapshot(t, w) != snap {
1220 t.Fatal("a refused rebind wrote")
1221 }
1222 key1, _ := hostclaim.LossRecordKey(sub1.Confirmation)
1223 _, tag, _ := w.raw.Store.Get(ctx, key1)
1224 if _, err := w.raw.Store.Put(ctx, key1, conflicted, tag); err != nil {
1225 t.Fatal(err)
1226 }
1227 if _, err := w.declare(sub2); err != nil {
1228 t.Fatalf("resume: %v", err)
1229 }
1230 if rec, _ := w.record(t); rec.Move.Rollback.TargetPhase != provision.RbTargetReleased || rec.Move.Rollback.Evidence != ref2 {
1231 t.Fatalf("rollback = %+v", rec.Move.Rollback)
1232 }
1233}
1234