amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostops
2
3import (
4 "context"
5 "errors"
6 "fmt"
7 "testing"
8 "time"
9
10 "github.com/carlosframework/platform/internal/hostclaim"
11 "github.com/carlosframework/platform/internal/provision"
12 "github.com/carlosframework/platform/internal/release"
13)
14
15func TestMoveRecoveryStartDurableFence(t *testing.T) {
16 for _, landed := range []bool{false, true} {
17 t.Run(map[bool]string{false: "absent", true: "applied"}[landed], func(t *testing.T) {
18 r := newResolveFixture(t, landed)
19 ctx := context.Background()
20 if r.effect(t).Inputs["record_base_seq"] != "0" {
21 t.Fatal("no durable base")
22 }
23 // Discard the issuing handle: only the durable evidence survives.
24 r.h = nil
25 run := func() (Outcome, error) {
26 var out Outcome
27 err := r.w.RunRecovery(ctx, func(ctx context.Context) error {
28 return runMoveRecovery(ctx, r.moving, func(ctx context.Context, g *MoveRecoveryGrant) error {
29 var err error
30 out, err = g.ResolveStart(ctx)
31 return err
32 })
33 })
34 return out, err
35 }
36 out, err := run()
37 if landed {
38 if err != nil || out != OutcomeComplete {
39 t.Fatalf("applied=%s,%v", out, err)
40 }
41 return
42 }
43 if !errors.Is(err, ErrMoveStartFenceUnproven) || out != OutcomeUncertain {
44 t.Fatalf("equal base=%s,%v", out, err)
45 }
46 cur, err := ReadRecord(ctx, r.raw.Store, r.raw.recordKey)
47 if err != nil {
48 t.Fatal(err)
49 }
50 if _, _, err := provision.WriteRecord(provision.WithRecordSequence(ctx), r.raw.Store, r.raw.recordKey, cur.Body, cur.Body, cur.ETag); err != nil {
51 t.Fatal(err)
52 }
53 out, err = run()
54 if err != nil || out != OutcomeNotApplied {
55 t.Fatalf("advanced=%s,%v", out, err)
56 }
57 if _, err := r.lost.Store.Put(ctx, r.raw.recordKey, r.lost.body, r.lost.etag); !errors.Is(err, release.ErrCASConflict) {
58 t.Fatalf("late put=%v", err)
59 }
60 })
61 }
62}
63
64func TestMoveRecoveryGrantCannotEscape(t *testing.T) {
65 r := newResolveFixture(t, false)
66 ctx := context.Background()
67 if err := runMoveRecovery(ctx, r.moving, func(context.Context, *MoveRecoveryGrant) error { return nil }); !errors.Is(err, ErrContract) {
68 t.Fatal(err)
69 }
70 var saved *MoveRecoveryGrant
71 if err := r.w.RunRecovery(ctx, func(ctx context.Context) error {
72 return runMoveRecovery(ctx, r.moving, func(_ context.Context, g *MoveRecoveryGrant) error { saved = g; return nil })
73 }); err != nil {
74 t.Fatal(err)
75 }
76 if _, err := saved.ResolveStart(ctx); !errors.Is(err, ErrPending) {
77 t.Fatalf("escaped=%v", err)
78 }
79 if _, err := (&MoveRecoveryGrant{}).ResolveStart(ctx); !errors.Is(err, ErrPending) {
80 t.Fatalf("zero grant=%v", err)
81 }
82 if err := r.w.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, r.moving, nil) }); !errors.Is(err, ErrContract) {
83 t.Fatal(err)
84 }
85}
86
87func TestMoveRecoveryRejectsChangedSequence(t *testing.T) {
88 r := newResolveFixture(t, false)
89 ctx := context.Background()
90 err := r.w.RunRecovery(ctx, func(ctx context.Context) error {
91 return runMoveRecovery(ctx, r.moving, func(ctx context.Context, g *MoveRecoveryGrant) error {
92 cur, err := ReadRecord(ctx, r.raw.Store, r.raw.recordKey)
93 if err != nil {
94 return err
95 }
96 if _, _, err := provision.WriteRecord(provision.WithRecordSequence(ctx), r.raw.Store, r.raw.recordKey, cur.Body, cur.Body, cur.ETag); err != nil {
97 return err
98 }
99 _, err = g.ResolveStart(ctx)
100 return err
101 })
102 })
103 if !errors.Is(err, release.ErrCASConflict) {
104 t.Fatalf("changed sequence=%v", err)
105 }
106}
107
108func TestMoveRecoveryGenericGrantCannotRelease(t *testing.T) {
109 r := newResolveFixture(t, false)
110 ctx := context.Background()
111 err := r.w.RunRecovery(ctx, func(ctx context.Context) error {
112 _, err := hostclaim.Update(ctx, r.data, r.moving, hostclaim.ReleaseMove(moveStartID, "ie-box"))
113 return err
114 })
115 if err == nil {
116 t.Fatal("generic recovery released moving claim")
117 }
118}
119
120func TestMoveRecoveryConvertUsesFenceAndReplaysDecision(t *testing.T) {
121 w := newC7World(t, nil)
122 w.mh = nil
123 ctx := context.Background()
124 run := func(ref hostclaim.FenceRef) error {
125 return w.control.RunRecovery(ctx, func(ctx context.Context) error {
126 return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Convert(ctx, ref) }, w.authority)
127 })
128 }
129 bad := w.ref
130 bad.SHA256 = "wrong"
131 if err := run(bad); err == nil {
132 t.Fatal("bad fence accepted")
133 }
134 if err := run(w.ref); err != nil {
135 t.Fatal(err)
136 }
137 if w.claim(t).Claim.Op.Kind != hostclaim.KindMoving {
138 t.Fatal("decision pass converted claim")
139 }
140 rec, err := provision.GetRecord(ctx, w.raw, "bab", "app", w.host)
141 if err != nil {
142 t.Fatal(err)
143 }
144 if rec.Move.State != provision.MoveRelocating {
145 t.Fatal("decision missing")
146 }
147 if err := run(w.ref); err != nil {
148 t.Fatal(err)
149 }
150 c := w.claim(t).Claim
151 if c.Op.Kind != hostclaim.KindRelocating || c.Op.Phase != "pinning" || c.Op.Exec != "none" {
152 t.Fatalf("conversion=%+v", c.Op)
153 }
154 if err := run(w.ref); err != nil {
155 t.Fatal(err)
156 }
157}
158
159func TestMoveRecoveryRecordCloseThenRelease(t *testing.T) {
160 w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
161 c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindLanding, hostclaim.PhaseCommitted, "target"
162 c.Obligations = nil
163 r.Region = "uk"
164 r.Move.State = provision.MoveCommitted
165 now := r.CreatedAt
166 r.Move.CommittedAt = &now
167 r.Move.OpenedAt = &now
168 r.Move.SourceParkedAt = &now
169 })
170 ctx := context.Background()
171 run := func(action func(context.Context, *MoveRecoveryGrant) error) error {
172 return w.control.RunRecovery(ctx, func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), action) })
173 }
174 if err := run(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Record(ctx, RecoveryMoveClose) }); err != nil {
175 t.Fatal(err)
176 }
177 if err := run(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Claim(ctx, RecoveryMoveRelease) }); err != nil {
178 t.Fatal(err)
179 }
180 if w.claim(t).Claim.Op.Kind != hostclaim.KindServing {
181 t.Fatal("not released")
182 }
183}
184
185func TestMoveRecoveryStartRequiresDurableBaseAndHonorsAppliedJournal(t *testing.T) {
186 for _, marker := range []string{"missing-base", "applied-effect", "issued"} {
187 t.Run(marker, func(t *testing.T) {
188 r := newResolveFixture(t, false)
189 ctx := context.Background()
190 cur, err := ReadRecord(ctx, r.raw.Store, r.raw.recordKey)
191 if err != nil {
192 t.Fatal(err)
193 }
194 if _, _, err := provision.WriteRecord(provision.WithRecordSequence(ctx), r.raw.Store, r.raw.recordKey, cur.Body, cur.Body, cur.ETag); err != nil {
195 t.Fatal(err)
196 }
197 _, err = r.w.retryAdmission(ctx, "bab", "app", "canonical", AdmissionObserved{}, func(a *AppAdmission) error {
198 e := a.Effects[moveStartID]
199 switch marker {
200 case "missing-base":
201 delete(e.Inputs, "record_base_seq")
202 case "applied-effect":
203 e.ResultReference = moveStartID
204 case "issued":
205 e.State = "issued"
206 }
207 a.Effects[moveStartID] = e
208 return nil
209 })
210 if err != nil {
211 t.Fatal(err)
212 }
213 var out Outcome
214 err = r.w.RunRecovery(ctx, func(ctx context.Context) error {
215 return runMoveRecovery(ctx, r.moving, func(ctx context.Context, g *MoveRecoveryGrant) error {
216 var err error
217 out, err = g.ResolveStart(ctx)
218 return err
219 })
220 })
221 switch marker {
222 case "missing-base":
223 if !errors.Is(err, ErrMoveStartFenceUnproven) {
224 t.Fatal(err)
225 }
226 case "applied-effect":
227 if err != nil || out != OutcomeComplete {
228 t.Fatalf("%s,%v", out, err)
229 }
230 case "issued":
231 if !errors.Is(err, ErrPending) {
232 t.Fatal(err)
233 }
234 }
235 })
236 }
237}
238
239func TestMoveRecoveryContextCannotMintAfterRunReturns(t *testing.T) {
240 r := newResolveFixture(t, false)
241 var saved context.Context
242 if err := r.w.RunRecovery(context.Background(), func(ctx context.Context) error { saved = ctx; return nil }); err != nil {
243 t.Fatal(err)
244 }
245 if err := runMoveRecovery(saved, r.moving, func(context.Context, *MoveRecoveryGrant) error { return nil }); !errors.Is(err, ErrContract) {
246 t.Fatalf("escaped recovery context=%v", err)
247 }
248}
249
250func TestMoveRecoveryCloseWaitsForObligations(t *testing.T) {
251 w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
252 c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindLanding, hostclaim.PhaseCommitted, "target"
253 r.Region = "uk"
254 r.Move.State = provision.MoveCommitted
255 now := r.CreatedAt
256 r.Move.CommittedAt = &now
257 r.Move.OpenedAt = &now
258 r.Move.SourceParkedAt = &now
259 })
260 err := w.control.RunRecovery(context.Background(), func(ctx context.Context) error {
261 return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Record(ctx, RecoveryMoveClose) })
262 })
263 if !errors.Is(err, ErrPending) {
264 t.Fatalf("outstanding parks=%v", err)
265 }
266}
267
268func TestMoveRecoveryGrantValueCopyExpires(t *testing.T) {
269 r := newResolveFixture(t, false)
270 ctx := context.Background()
271 var saved MoveRecoveryGrant
272 if err := r.w.RunRecovery(ctx, func(ctx context.Context) error {
273 return runMoveRecovery(ctx, r.moving, func(_ context.Context, g *MoveRecoveryGrant) error { saved = *g; return nil })
274 }); err != nil {
275 t.Fatal(err)
276 }
277 if _, err := saved.ResolveStart(ctx); !errors.Is(err, ErrPending) {
278 t.Fatalf("copied grant=%v", err)
279 }
280}
281
282func TestMoveRecoveryHookCannotRemint(t *testing.T) {
283 r := newResolveFixture(t, false)
284 err := r.w.RunRecovery(context.Background(), func(ctx context.Context) error {
285 return runMoveRecovery(ctx, r.moving, func(ctx context.Context, g *MoveRecoveryGrant) error {
286 if err := runMoveRecovery(ctx, r.moving, func(context.Context, *MoveRecoveryGrant) error { return nil }); !errors.Is(err, ErrContract) {
287 t.Fatalf("hook reminted=%v", err)
288 }
289 return nil
290 })
291 })
292 if err != nil {
293 t.Fatal(err)
294 }
295}
296
297func TestMoveRecoveryRejectsTargetParkedAction(t *testing.T) {
298 w := newJ17World(t, nil)
299 ctx := context.Background()
300 before, tag, err := w.raw.Get(ctx, w.key)
301 if err != nil {
302 t.Fatal(err)
303 }
304 err = w.control.RunRecovery(ctx, func(ctx context.Context) error {
305 return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error {
306 return g.Record(ctx, MoveRecoveryRecordAction("target-parked"))
307 })
308 })
309 if !errors.Is(err, ErrIdentity) {
310 t.Fatalf("unrecognized action=%v", err)
311 }
312 after, afterTag, err := w.raw.Get(ctx, w.key)
313 if err != nil {
314 t.Fatal(err)
315 }
316 if string(before) != string(after) || tag != afterTag {
317 t.Fatal("unrecognized action changed record")
318 }
319}
320
321func TestMoveRecoveryGrantOverwriteCannotRedirectRevocation(t *testing.T) {
322 r := newResolveFixture(t, false)
323 err := r.w.RunRecovery(context.Background(), func(ctx context.Context) error {
324 var saved MoveRecoveryGrant
325 if err := runMoveRecovery(ctx, r.moving, func(_ context.Context, g *MoveRecoveryGrant) error { saved = *g; *g = MoveRecoveryGrant{}; return nil }); err != nil {
326 return err
327 }
328 if _, err := saved.ResolveStart(ctx); !errors.Is(err, ErrPending) {
329 t.Fatalf("overwritten callback grant=%v", err)
330 }
331 return nil
332 })
333 if err != nil {
334 t.Fatal(err)
335 }
336}
337
338type moveRecoveryBumpAuthority struct {
339 c7ReplayAuthority
340 bump func()
341}
342
343func (a *moveRecoveryBumpAuthority) CheckOneBoxRegion(ctx context.Context, source, region string) error {
344 if a.bump != nil {
345 f := a.bump
346 a.bump = nil
347 f()
348 }
349 return a.c7ReplayAuthority.CheckOneBoxRegion(ctx, source, region)
350}
351
352func TestMoveRecoveryConvertRefusesSequenceBumpDuringFenceCheck(t *testing.T) {
353 w := newC7World(t, nil)
354 ctx := context.Background()
355 run := func(a hostclaim.FenceAuthority) error {
356 return w.control.RunRecovery(ctx, func(ctx context.Context) error {
357 return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Convert(ctx, w.ref) }, a)
358 })
359 }
360 if err := run(w.authority); err != nil {
361 t.Fatal(err)
362 }
363 authority := &moveRecoveryBumpAuthority{c7ReplayAuthority: w.authority, bump: func() {
364 cur, err := ReadRecord(ctx, w.raw, w.key)
365 if err != nil {
366 t.Fatal(err)
367 }
368 if _, _, err := provision.WriteRecord(provision.WithRecordSequence(ctx), w.raw, w.key, cur.Body, cur.Body, cur.ETag); err != nil {
369 t.Fatal(err)
370 }
371 }}
372 if err := run(authority); !errors.Is(err, release.ErrCASConflict) {
373 t.Fatalf("sequence changed during verification=%v", err)
374 }
375 if w.claim(t).Claim.Op.Kind != hostclaim.KindMoving {
376 t.Fatal("converted at stale record sequence")
377 }
378}
379
380func TestMoveRecoveryPassUsesTypedHookOnlyInsideRunRecovery(t *testing.T) {
381 f := newResolveFixture(t, true)
382 noWork := func(context.Context) error { return nil }
383 wrongKind := func(context.Context, hostclaim.Observed) error { t.Fatal("move reached generic recovery"); return nil }
384 calls := 0
385 r := Recovery{Controller: Controller{Store: f.raw, Control: f.w, Authority: &f.authority, Now: func() time.Time { return f.now }, NewID: func() string { return "recovery" }}, Purge: wrongKind, Placement: wrongKind, SweepRecords: noWork, SweepPurges: noWork, SweepAdmissions: noWork}
386 r.Moves = func(ctx context.Context, g *MoveRecoveryGrant) error {
387 calls++
388 if g.Observation().OpID != f.moving.Claim.Op.ID {
389 t.Fatal("wrong move grant")
390 }
391 outcome, err := g.ResolveStart(ctx)
392 if err != nil || outcome != OutcomeComplete {
393 t.Fatalf("settle=%s,%v", outcome, err)
394 }
395 return nil
396 }
397 if err := r.Pass(t.Context()); !errors.Is(err, ErrContract) || calls != 0 {
398 t.Fatalf("outside recovery: %v calls=%d", err, calls)
399 }
400 missing := r
401 missing.Moves = nil
402 if err := f.w.RunRecovery(t.Context(), missing.Pass); err != nil {
403 t.Fatalf("missing hook: %v", err)
404 }
405 var saved context.Context
406 if err := f.w.RunRecovery(t.Context(), func(ctx context.Context) error { saved = ctx; return r.Pass(ctx) }); err != nil {
407 t.Fatal(err)
408 }
409 if calls != 1 {
410 t.Fatalf("calls=%d", calls)
411 }
412 if err := r.Pass(saved); !errors.Is(err, ErrContract) || calls != 1 {
413 t.Fatalf("retained pass context: %v calls=%d", err, calls)
414 }
415}
416
417func TestMoveRecoveryReleaseRequiresMatchingOutcomeAndExecutor(t *testing.T) {
418 for _, kind := range []string{hostclaim.KindLanding, hostclaim.KindRestoring, hostclaim.KindReverting, hostclaim.KindMoving} {
419 for _, outcome := range []string{"done", "rolled_back", "aborted"} {
420 for _, foreign := range []bool{false, true} {
421 t.Run(kind+"/"+outcome+fmt.Sprint(foreign), func(t *testing.T) {
422 valid := !foreign && ((kind == hostclaim.KindLanding && outcome == "done") || ((kind == hostclaim.KindRestoring || kind == hostclaim.KindReverting) && outcome == "rolled_back") || (kind == hostclaim.KindMoving && outcome == "aborted"))
423 w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, _ *AppAdmission) {
424 c.Op.Kind, c.Op.Phase, c.Op.Exec = kind, hostclaim.PhaseOpened, c.Op.Intent["source"]
425 if kind == hostclaim.KindLanding {
426 c.Op.Phase, c.Op.Exec = hostclaim.PhaseCommitted, c.Op.Intent["target"]
427 }
428 if kind == hostclaim.KindMoving {
429 c.Op.Phase = hostclaim.PhaseAborting
430 }
431 if foreign {
432 c.Op.Exec = "foreign"
433 }
434 c.Obligations = nil
435 r.LastMove = &provision.MoveOutcome{ID: r.Move.ID, Outcome: outcome}
436 r.Move = nil
437 })
438 before := w.claim(t)
439 err := w.control.RunRecovery(t.Context(), func(ctx context.Context) error {
440 return runMoveRecovery(ctx, before, func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Claim(ctx, RecoveryMoveRelease) })
441 })
442 if valid && err != nil {
443 t.Fatal(err)
444 }
445 if !valid && (err == nil || w.claim(t).ETag != before.ETag) {
446 t.Fatalf("invalid release: %v", err)
447 }
448 })
449 }
450 }
451 }
452}
453