carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "testing"
9
10 "github.com/carlosframework/platform/internal/appconfig"
11 "github.com/carlosframework/platform/internal/hostclaim"
12 "github.com/carlosframework/platform/internal/hostguard"
13 "github.com/carlosframework/platform/internal/moves"
14 "github.com/carlosframework/platform/internal/park"
15 "github.com/carlosframework/platform/internal/provision"
16 "github.com/carlosframework/platform/internal/registry"
17 "github.com/carlosframework/platform/internal/release"
18)
19
20// The cut guard (Paul's ruling, 2026-10-11): the source refuses to start a
21// cut unless the target's latest report states that its box can land the
22// move and roll it back (T13), and unless the source itself can recover.
23// The refusal comes before anything stops, the writer's lease is taken or
24// the claim is sealed, and the move stays ready with the reason recorded.
25
26// errStandIn is what a harness stand-in answers. The stand-ins arm a box
27// so that the arming predicates hold, as they do once the wake-source gate
28// opens (movesWakeSourceRegistered, Plan 5 Task 9b);
29// a test that reaches one has run a step its harness never wired.
30var errStandIn = errors.New("harness stand-in: this hook is not wired in this harness")
31
32func standIn(name string) error { return fmt.Errorf("%s: %w", name, errStandIn) }
33
34// armStandIns fills every hook landingArmed, rollbackTargetArmed and
35// recoverySourceArmed read that c lacks, with the park fakes (as
36// armParkFakes arms them, for role) and a stand-in for each other hook. A hook c already has is
37// kept, so a harness's own fakes win.
38func armStandIns(c *moveCfg, s release.Store, role string) {
39 if c.Guard.DataDir == "" {
40 c.Guard = hostguard.Guard{DataDir: c.DataDir}
41 }
42 if c.Local.DataDir == "" {
43 c.Local = park.Local{DataDir: c.DataDir, Box: c.Box}
44 }
45 if c.ParkLeases == nil {
46 c.ParkLeases = claimParkLeases(&fakeParkLeases{})
47 }
48 if c.ParkView == nil {
49 c.ParkView = recordView(s, c.Region, role)
50 }
51 if c.ParkAuthorize == nil {
52 c.ParkAuthorize = allowParkAuthorize
53 }
54 if c.ParkRunHeld == nil {
55 c.ParkRunHeld = movesRunHeld
56 }
57 if c.ParkLocalAuthorize == nil {
58 c.ParkLocalAuthorize = allowLocalAuthorize
59 }
60 if c.OwnLease == nil {
61 c.OwnLease = func(context.Context, string, string) (park.HeldLease, error) { return nil, standIn("OwnLease") }
62 }
63 if c.RenderHeld == nil {
64 c.RenderHeld = func(context.Context, release.Store, registry.Route, string, *provision.MoveInputs, func(string) error) (appconfig.HeldRender, error) {
65 return appconfig.HeldRender{}, standIn("RenderHeld")
66 }
67 }
68 if c.ReplaceTimers == nil {
69 c.ReplaceTimers = func(context.Context, string, []registry.Timer) error { return standIn("ReplaceTimers") }
70 }
71 if c.SetReplication == nil {
72 c.SetReplication = func(string, string) error { return standIn("SetReplication") }
73 }
74 if c.SetNextDue == nil {
75 c.SetNextDue = func(string, int64) error { return standIn("SetNextDue") }
76 }
77 if c.RemoveVerifyEntry == nil {
78 c.RemoveVerifyEntry = func(context.Context, release.Store, string) error { return standIn("RemoveVerifyEntry") }
79 }
80 if c.WriteGen == nil {
81 c.WriteGen = func(string, string) error { return standIn("WriteGen") }
82 }
83 if c.ReleaseReady == nil {
84 c.ReleaseReady = func(context.Context, release.Store, provision.Record, string) moves.Check {
85 return moves.Check{Reason: standIn("ReleaseReady").Error()}
86 }
87 }
88 if c.HandHold == nil {
89 c.HandHold = func(string, string, string, string, func() error) (bool, error) { return false, standIn("HandHold") }
90 }
91 if c.ReleaseHeld == nil {
92 c.ReleaseHeld = func(string, string) (bool, error) { return false, standIn("ReleaseHeld") }
93 }
94 if c.Timers == nil {
95 c.Timers = func(string) ([]registry.Timer, error) { return nil, standIn("Timers") }
96 }
97 if role != roleTarget {
98 return
99 }
100 if c.DeleteHeld == nil {
101 c.DeleteHeld = func(string, string) (bool, error) { return false, standIn("DeleteHeld") }
102 }
103 if c.Fence == nil {
104 c.Fence = func(context.Context, string, uint64) error { return standIn("Fence") }
105 }
106 if c.TimersDrained == nil {
107 c.TimersDrained = func(context.Context, string) error { return standIn("TimersDrained") }
108 }
109 if c.Evidence == nil {
110 c.Evidence = func(context.Context, string, string, []string) (moves.StoppedReport, error) {
111 return moves.StoppedReport{}, standIn("Evidence")
112 }
113 }
114 if c.FinalCheckpoint == nil {
115 c.FinalCheckpoint = func(context.Context, string, string, hostclaim.Holder, func(context.Context) error) error {
116 return standIn("FinalCheckpoint")
117 }
118 }
119 if c.LocalPosition == nil {
120 c.LocalPosition = func(string) (uint64, error) { return 0, standIn("LocalPosition") }
121 }
122}
123
124// unarmedTarget is the cut harness's target with the landing or the
125// rollback left unwired, as every box is in production until the
126// wake-source gate opens (Plan 5 Task 9b).
127func (h *cutHarness) unarmedTarget(t *testing.T, unarm func(*moveCfg)) moveCfg {
128 t.Helper()
129 c := h.target()
130 if !landingArmed(c) || !rollbackTargetArmed(c) {
131 t.Fatal("fixture: the harness's target is not armed")
132 }
133 unarm(&c)
134 if landingArmed(c) && rollbackTargetArmed(c) {
135 t.Fatal("fixture: the target is still armed")
136 }
137 return c
138}
139
140// nothingStopped fails unless the move is ready again with reason recorded
141// and the pass did nothing irreversible: no hold, fence, drain, stop
142// evidence (which reads the writer's lease) or final checkpoint, no staging
143// lease taken and no status set since the counts before the pass, and the
144// claim still the moving op under the source with nothing sealed.
145func (h *cutHarness) nothingStopped(t *testing.T, reason string, acquired, statuses int) {
146 t.Helper()
147 m := h.move(t)
148 if m.State != provision.MoveReady || m.CutPhase != "" || m.Inputs != nil || m.CutAt != nil || m.Reason != reason {
149 t.Fatalf("not refused back to ready with %q: %+v", reason, m)
150 }
151 if h.ev.calls != 0 || len(h.holds) != 0 || h.fences != 0 || h.drains != 0 || h.checkpoints != 0 || len(h.locations) != 0 {
152 t.Fatalf("something stopped: evidence %d, holds %v, fences %d, drains %d, checkpoints %d, leases read %v",
153 h.ev.calls, h.holds, h.fences, h.drains, h.checkpoints, h.locations)
154 }
155 if got := h.lease.acquired; got != acquired {
156 t.Fatalf("the refusal took the staging lease: %d acquisitions, want %d", got, acquired)
157 }
158 if len(h.statuses) != statuses {
159 t.Fatalf("the source's row changed status: %v", h.statuses[statuses:])
160 }
161 c := h.claim(t)
162 if c.Op.Exec != "ie-box" || c.Op.Kind != hostclaim.KindMoving || c.Op.Phase != hostclaim.PhaseMoving {
163 t.Fatalf("the claim moved: %+v", c.Op)
164 }
165 if _, err := journalledSeal(c); err == nil {
166 t.Fatal("the claim carries a seal")
167 }
168}
169
170// The harnesses arm both sides, so the cut runs exactly as before: the
171// target's report states T13, the source's recovery is armed.
172func TestCutGuardBothArmedCutsAsBefore(t *testing.T) {
173 h := newCutHarness(t)
174 h.ready(t)
175 if r := h.targetReport(t); !r.Armed() || r.Checks[moves.CheckArmed] != (moves.Check{OK: true}) {
176 t.Fatalf("the armed target's report: %+v", r.Checks[moves.CheckArmed])
177 }
178 if !recoverySourceArmed(h.cfg) {
179 t.Fatal("fixture: the source's recovery is not armed")
180 }
181 h.cut(t)
182 h.pass(t)
183 if m := h.move(t); m.State != provision.MoveLanding || m.CutPhase != provision.CutFlipped || m.Reason != "" {
184 t.Fatalf("the armed cut did not flip: %+v", m)
185 }
186}
187
188// A target whose report says it cannot land or roll back is refused at
189// requested, before the source's round takes the staging lease: the move
190// is ready again, unpinned, with the reason, and nothing stopped. So is a
191// report from an older box, which has no T13 at all.
192func TestCutGuardRefusesAnUnarmedTarget(t *testing.T) {
193 t.Parallel()
194 ctx := context.Background()
195 for name, spoil := range map[string]func(t *testing.T, h *cutHarness){
196 "its landing is not armed": func(t *testing.T, h *cutHarness) {
197 h.targetPass(t, h.unarmedTarget(t, func(c *moveCfg) { c.OwnLease = nil }))
198 },
199 "its rollback is not armed": func(t *testing.T, h *cutHarness) {
200 h.targetPass(t, h.unarmedTarget(t, func(c *moveCfg) { c.DeleteHeld = nil }))
201 },
202 "an older box's report has no T13": func(t *testing.T, h *cutHarness) {
203 rec := h.read(t, p1Host)
204 key := moves.TargetReportKey(moves.ReportPrefix(rec.Account, rec.App, rec.Host, rec.Move.ID))
205 body, _, err := h.s.Get(ctx, key)
206 if err != nil {
207 t.Fatal(err)
208 }
209 var raw map[string]json.RawMessage
210 if err := json.Unmarshal(body, &raw); err != nil {
211 t.Fatal(err)
212 }
213 var checks map[string]json.RawMessage
214 if err := json.Unmarshal(raw["checks"], &checks); err != nil {
215 t.Fatal(err)
216 }
217 delete(checks, moves.CheckArmed)
218 raw["checks"], _ = json.Marshal(checks)
219 body, _ = json.Marshal(raw)
220 if err := moves.PutReplace(ctx, h.s, key, body); err != nil {
221 t.Fatal(err)
222 }
223 },
224 } {
225 t.Run(name, func(t *testing.T) {
226 h := newCutHarness(t)
227 h.ready(t)
228 spoil(t, h)
229 if h.targetReport(t).Armed() {
230 t.Fatal("fixture: the report still states T13")
231 }
232 h.cut(t)
233 acquired, statuses := h.lease.acquired, len(h.statuses)
234 h.pass(t)
235 h.nothingStopped(t, provision.RefusedTargetUnarmed, acquired, statuses)
236 })
237 }
238}
239
240// A source that cannot recover a failed cut refuses it the same way.
241func TestCutGuardRefusesAnUnarmedSource(t *testing.T) {
242 t.Parallel()
243 for name, unarm := range map[string]func(*moveCfg){
244 "no recovery open": func(c *moveCfg) { c.ReleaseHeld = nil },
245 "no hand-over": func(c *moveCfg) { c.HandHold = nil },
246 "no landing for return": func(c *moveCfg) { c.ParkRunHeld = nil },
247 } {
248 t.Run(name, func(t *testing.T) {
249 h := newCutHarness(t)
250 h.ready(t)
251 unarm(&h.cfg)
252 if recoverySourceArmed(h.cfg) {
253 t.Fatal("fixture: the source is still armed")
254 }
255 h.cut(t)
256 acquired, statuses := h.lease.acquired, len(h.statuses)
257 h.pass(t)
258 h.nothingStopped(t, provision.RefusedSourceUnarmed, acquired, statuses)
259 })
260 }
261}
262
263// At admitted (a record an older source admitted without the guard), the
264// guard refuses before the step to stopping. The target writes its report
265// while the move is ready, so the unarmed one is published before the cut.
266func TestCutGuardRefusesAtAdmitted(t *testing.T) {
267 ctx := context.Background()
268 h := newCutHarness(t)
269 h.ready(t)
270 h.targetPass(t, h.unarmedTarget(t, func(c *moveCfg) { c.SetNextDue = nil }))
271 h.cut(t)
272 g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}
273 now := h.clock.Now()
274 if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveAdmit(g, now, 120, moves.CutDeadline(now, 120), 3)); err != nil {
275 t.Fatal(err)
276 }
277 acquired, statuses := h.lease.acquired, len(h.statuses)
278 h.pass(t)
279 h.nothingStopped(t, provision.RefusedTargetUnarmed, acquired, statuses)
280}
281
282// The refusal stands without a loop: an automatic cut refused because the
283// source cannot recover is not pinned again or cut again while that holds,
284// so later passes write nothing. Once the source is armed, the next passing
285// round pins again and the automatic cut proceeds.
286func TestCutGuardRefusalStandsUntilArmed(t *testing.T) {
287 h := newCutHarness(t)
288 h.start(t, "auto")
289 rootHolder(t, h.phase1Harness)
290 armed := h.cfg
291 h.cfg.ReleaseHeld = nil
292 h.ev.stopped = false // stop at stopping once armed, to look at it
293 h.pass(t)
294 h.targetPass(t, h.target())
295 for i := 0; i < 3; i++ {
296 h.pass(t)
297 }
298 refused := h.move(t)
299 if refused.State != provision.MoveReady || refused.Inputs != nil || refused.Reason != provision.RefusedSourceUnarmed {
300 t.Fatalf("the automatic cut was not refused: %+v", refused)
301 }
302 h.targetPass(t, h.target())
303 h.pass(t)
304 h.pass(t)
305 if m := h.move(t); m.UpdatedAt != refused.UpdatedAt || m.Inputs != nil || m.Reason != provision.RefusedSourceUnarmed {
306 t.Fatalf("passes after the refusal wrote the record: %+v", m)
307 }
308 if h.ev.calls != 0 || len(h.holds) != 0 {
309 t.Fatalf("something stopped: evidence %d, holds %v", h.ev.calls, h.holds)
310 }
311 h.cfg = armed
312 h.pass(t) // pins again and requests the cut
313 h.pass(t) // admits and starts stopping
314 if m := h.move(t); m.State != provision.MoveCutting || m.CutPhase != provision.CutStopping || m.Reason != "" {
315 t.Fatalf("the armed source did not cut: %+v", m)
316 }
317}
318
319// With the switch off, a requested cut on an unarmed source is left as main
320// leaves it: nothing refuses, nothing is written.
321func TestCutGuardSwitchOffIsMain(t *testing.T) {
322 h := newCutHarness(t)
323 h.ready(t)
324 h.cut(t)
325 before := h.move(t)
326 h.cfg.Enabled = false
327 h.cfg.ReleaseHeld = nil
328 h.pass(t)
329 h.pass(t)
330 if m := h.move(t); m.State != provision.MoveCutting || m.CutPhase != provision.CutRequested || m.Reason != "" || m.UpdatedAt != before.UpdatedAt {
331 t.Fatalf("the switch-off pass wrote the record: %+v", m)
332 }
333 if h.ev.calls != 0 || len(h.holds) != 0 {
334 t.Fatalf("something stopped: evidence %d, holds %v", h.ev.calls, h.holds)
335 }
336}
337
338// T13 gates readiness (§7.3): an unarmed target's report fails T13 with its
339// sentence, and the move stays preparing, so `carlos move status` shows why
340// before anyone asks to cut. Armed, it passes and the move is ready.
341func TestTargetReportStatesItsArming(t *testing.T) {
342 h := newCutHarness(t)
343 h.start(t, "manual")
344 rootHolder(t, h.phase1Harness)
345 h.pass(t)
346 h.targetPass(t, h.unarmedTarget(t, func(c *moveCfg) { c.ParkView = nil }))
347 r := h.targetReport(t)
348 if f := failing(r); len(f) != 1 || f[moves.CheckArmed] != t13NotArmed {
349 t.Fatalf("an unarmed target's failing checks: %v", f)
350 }
351 h.pass(t)
352 if m := h.move(t); m.State != provision.MovePreparing {
353 t.Fatalf("an unarmed target's move: %+v", m)
354 }
355 h.targetPass(t, h.target())
356 if f := failing(h.targetReport(t)); len(f) != 0 {
357 t.Fatalf("an armed target's failing checks: %v", f)
358 }
359 h.pass(t)
360 if m := h.move(t); m.State != provision.MoveReady || m.Inputs == nil {
361 t.Fatalf("an armed target's move: %+v", m)
362 }
363}
364
365// Fix round 1: a refusal whose record write loses to another writer still
366// ends the step. Here another writer takes the move back to ready just
367// before the refusal's write, so the refusal's CAS conflicts; the pass must
368// not carry on with the cut (the source's round and its staging lease at
369// requested, the step to stopping at admitted). The next pass re-reads.
370func TestCutGuardRefusalEndsTheStepOnAConflict(t *testing.T) {
371 t.Parallel()
372 ctx := context.Background()
373 for _, phase := range []string{provision.CutRequested, provision.CutAdmitted} {
374 t.Run(phase, func(t *testing.T) {
375 h := newCutHarness(t)
376 h.ready(t)
377 h.targetPass(t, h.unarmedTarget(t, func(c *moveCfg) { c.OwnLease = nil }))
378 h.cut(t)
379 g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}
380 if phase == provision.CutAdmitted {
381 now := h.clock.Now()
382 if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveAdmit(g, now, 120, moves.CutDeadline(now, 120), 3)); err != nil {
383 t.Fatal(err)
384 }
385 }
386 inner, conflicted := h.cfg.cutFault, false
387 h.cfg.cutFault = func(point string) error {
388 if point == "refusal" && !conflicted {
389 conflicted = true
390 if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveBackToReady(g)); err != nil {
391 t.Errorf("the conflicting write: %v", err)
392 }
393 }
394 return inner(point)
395 }
396 acquired, statuses := h.lease.acquired, len(h.statuses)
397 h.pass(t)
398 if !conflicted {
399 t.Fatal("the pass never reached the refusal")
400 }
401 // The other writer's decision stands: ready, unpinned, and
402 // no reason, since its write carried none.
403 h.nothingStopped(t, "", acquired, statuses)
404 })
405 }
406}
407
408// Fix round 1 (ruling on concern 2): the scoped-holder refusal (ruling
409// T5-lease) ends the step whatever its write returns, as the cut guard's
410// does. Another writer takes the move back to ready just before the
411// refusal's write, so its CAS conflicts; the pass ends there. At admitted
412// the step must not go on to the cut guard or to stopping (the target is
413// left unarmed so that a step that carried on would reach the guard's
414// "refusal" point); at requested it must not go on to the admission. The
415// staging lease the requested round takes before the holder check is
416// main's order (ruling T5-lease) and is not this refusal's.
417func TestCutHolderRefusalEndsTheStepOnAConflict(t *testing.T) {
418 t.Parallel()
419 ctx := context.Background()
420 for _, phase := range []string{provision.CutRequested, provision.CutAdmitted} {
421 t.Run(phase, func(t *testing.T) {
422 h := newCutHarness(t)
423 h.scoped = true
424 h.ready(t)
425 if phase == provision.CutAdmitted {
426 h.targetPass(t, h.unarmedTarget(t, func(c *moveCfg) { c.OwnLease = nil }))
427 }
428 h.cut(t)
429 g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc}
430 if phase == provision.CutAdmitted {
431 now := h.clock.Now()
432 if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveAdmit(g, now, 120, moves.CutDeadline(now, 120), 3)); err != nil {
433 t.Fatal(err)
434 }
435 }
436 inner, conflicted := h.cfg.cutFault, false
437 var after []string
438 h.cfg.cutFault = func(point string) error {
439 if conflicted {
440 after = append(after, point)
441 }
442 if point == "holder-refusal" && !conflicted {
443 conflicted = true
444 if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, provision.MoveBackToReady(g)); err != nil {
445 t.Errorf("the conflicting write: %v", err)
446 }
447 }
448 return inner(point)
449 }
450 acquired, statuses := h.lease.acquired, len(h.statuses)
451 h.pass(t)
452 if !conflicted {
453 t.Fatal("the pass never reached the holder refusal")
454 }
455 if len(after) != 0 {
456 t.Fatalf("the step carried on after the conflicting refusal: %v", after)
457 }
458 if phase == provision.CutRequested {
459 acquired = h.lease.acquired // the round's own, before the holder check
460 }
461 h.nothingStopped(t, "", acquired, statuses)
462 if m := h.move(t); m.CutAt != nil || m.CutBudgetS != 0 {
463 t.Fatalf("the cut was admitted: %+v", m)
464 }
465 })
466 }
467}
468