| 1 | //go:build unix |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "encoding/json" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/hostbox" |
| 12 | "github.com/carlosframework/platform/internal/hostclaim" |
| 13 | "github.com/carlosframework/platform/internal/hostguard" |
| 14 | "github.com/carlosframework/platform/internal/registry" |
| 15 | ) |
| 16 | |
| 17 | // Carry 9 (Task 9 review, ruling 82): a relocation completes on box-1 with |
| 18 | // region HA on; box-1 restarts with CARLOS_REGION_HA off. One scenario, |
| 19 | // with no agent restart between its halves, over one control bucket, one |
| 20 | // guard journal, one registry and one activator: |
| 21 | // |
| 22 | // 1. the probe stop: the production probe-stop provider fences the probe |
| 23 | // session under its persisted owner and root's writer-session stop |
| 24 | // lands in the guard journal; the serving CAS records the completion; |
| 25 | // 2. the completion pass: NewRelocationRuntime with claim mode alone |
| 26 | // builds the completion-only runtime, whose pass discharges through |
| 27 | // main's own DischargeCompleted, releases the hold, lifts the owned |
| 28 | // fence and marks the record, so the barrier opens; |
| 29 | // 3. the first serving wake: the wake runtime (ClaimWakeAssembly.Build) |
| 30 | // wired as OpenProductionClaimAssembly wires it with moves C1 off |
| 31 | // (wireProductionAdmission, N6 in front) takes the predecessor stop |
| 32 | // Prerequisites takes, which re-stops nothing, then admits through |
| 33 | // the runtime's own Admit, which hands root the probe's receipt as the |
| 34 | // prior. |
| 35 | // |
| 36 | // The production constructor itself cannot run here in agent mode: it |
| 37 | // dials root at the fixed controlsock.Path, chowns its state directory to |
| 38 | // the carlos uid and needs a cgroup manager. Checklist item 12 is that run. |
| 39 | func TestCompletionPassThenFirstServingWakeWithRegionHAOff(t *testing.T) { |
| 40 | t.Parallel() |
| 41 | ctx := t.Context() |
| 42 | f := newRecoveryFixture(t) |
| 43 | const box, op = "box-1", "rel-1" |
| 44 | local := claimStopLocal{Actor: hostbox.Actor{Box: box, Enrollment: "enrollment-1", Revision: 7, Binding: "7", Region: "ie"}, BootID: "boot-0", ControllerID: "controller-0", PID: 4000, ProcessStart: "start-0"} |
| 45 | |
| 46 | // The relocating claim at probing on box-1, its probe passed with |
| 47 | // session 12 at install stamp 9. |
| 48 | // Written whole, as the stop-probe fixture's claim is: the transitions |
| 49 | // that lead here are hostrelocate's and are tested there. |
| 50 | probe, _ := json.Marshal(hostclaim.Probe{Box: box, Session: 12, At: f.now}) |
| 51 | created := f.read() |
| 52 | c := *created.Claim |
| 53 | holder := *c.Holder |
| 54 | holder.Stamp, holder.Session, holder.Generation, holder.WriterBox = 9, 12, "P1", box |
| 55 | holder.Admissions = []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}} |
| 56 | c.Holder = &holder |
| 57 | c.Gen = 30 // the history behind stamp 9 and session 12 |
| 58 | c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: op, Phase: hostclaim.PhaseRelocatingProbing, Exec: box, Participants: []string{"src", box}, |
| 59 | Intent: map[string]string{"region": "ie", "source_box": "src", "target_box": box}, Journal: map[string]string{"probe": string(probe)}} |
| 60 | key, _ := hostclaim.Key(recoveryHost) |
| 61 | if _, err := f.claims.Put(ctx, key, mustJSON(t, &c), created.ETag); err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | o := f.read() |
| 65 | if err := f.guard.Activate(ctx, recoveryHost, box, nil); err != nil { |
| 66 | t.Fatal(err) |
| 67 | } |
| 68 | |
| 69 | // 1. The probe stop, root's writer-session stop standing as Wake.Stop. |
| 70 | h := o.Claim.Holder |
| 71 | target := hostguard.FenceTarget{FenceID: "fence-1", Region: "ie", Host: recoveryHost, Incarnation: h.Incarnation, Stamp: 9, Session: 12, Op: op, Phase: hostclaim.PhaseRelocatingProbing, |
| 72 | ObligationID: hostguard.WriterStopObligation(12), Box: box, Enrollment: local.Actor.Enrollment, BootID: local.BootID, ControllerID: local.ControllerID, ClaimETag: o.ETag, DirectoryRevision: "7", RosterRevision: 7} |
| 73 | at := f.now.Add(time.Minute) |
| 74 | receipt := mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r1", Kind: StopReceiptStopped, Region: "ie", ClaimETag: o.ETag, ObligationID: target.ObligationID, FenceID: "fence-1", Host: recoveryHost, Incarnation: h.Incarnation, |
| 75 | InstallationStamp: 9, WriterSession: 12, OperationID: op, Phase: hostclaim.PhaseRelocatingProbing, Box: box, Enrollment: local.Actor.Enrollment, BootID: local.BootID, ControllerID: local.ControllerID, DirectoryRevision: testReceiptNumeral(target), Binding: "7", EvidenceAt: at}) |
| 76 | var order []string |
| 77 | act := newStopProbeActivator(&order) |
| 78 | probeAssembly := ClaimAssembly{Claims: f.claims, Guard: f.guard, stopLocal: func(context.Context) (claimStopLocal, error) { return local, nil }, |
| 79 | Wake: &ClaimWakeAssembly{Stop: func(ctx context.Context, _ hostclaim.Observed) error { |
| 80 | if _, err := f.guard.BeginFence(ctx, target); err != nil { |
| 81 | return err |
| 82 | } |
| 83 | return f.guard.CompleteFence(ctx, target, receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }) |
| 84 | }}} |
| 85 | stop, err := probeAssembly.relocationStopProbe(RelocationAssemblyConfig{Activator: act, DataDir: f.guard.DataDir})(ctx, o) |
| 86 | if err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | if !act.svc.Fenced(recoveryHost) { |
| 90 | t.Fatal("the probe stop installed no fence") |
| 91 | } |
| 92 | // The serving CAS, as CompleteRelocation leaves it. |
| 93 | done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: op, Stop: stop, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| 94 | c.Gen = 31 |
| 95 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: box, Participants: []string{box}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} |
| 96 | if _, err := f.claims.Put(ctx, key, mustJSON(t, &c), o.ETag); err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | reg, err := registry.Open(t.TempDir() + "/registry.db") |
| 100 | if err != nil { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | t.Cleanup(func() { reg.Close() }) |
| 104 | if err := reg.Upsert(registry.Route{Host: recoveryHost, Status: "relocating", HoldOp: op, Socket: "/run/shop.sock"}); err != nil { |
| 105 | t.Fatal(err) |
| 106 | } |
| 107 | |
| 108 | // 2. The completion pass after the restart with region HA off: claim |
| 109 | // mode alone, main's own participation owner and discharge. |
| 110 | identity := handoverRecoveryIdentity(t, local.Actor) |
| 111 | bound := &ClaimAssembly{Claims: f.claims, Guard: f.guard, stopLocal: func(context.Context) (claimStopLocal, error) { return local, nil }} |
| 112 | owner := &productionClaimLocal{legacyParticipation: true, Resources: ClaimRuntimeResources{Mode: "agent", DataDir: f.guard.DataDir}, Identity: identity, Store: f.claims, |
| 113 | slots: map[string]*claimParticipation{}, assembly: bound, lifetime: ctx} |
| 114 | claimOnly := func(k string) string { |
| 115 | if k == "CARLOS_HOST_CLAIM" { |
| 116 | return "1" |
| 117 | } |
| 118 | return "" |
| 119 | } |
| 120 | r, err := NewRelocationRuntime(ctx, claimOnly, f.claims, owner, ClaimProviders{}, reg, act, relocationRuntimeInputs{}) |
| 121 | if err != nil || r == nil || r.Worker != nil || r.Participant != nil || r.Convert != nil { |
| 122 | t.Fatalf("not the completion-only runtime: %+v %v", r, err) |
| 123 | } |
| 124 | r.once(ctx, f.claims) |
| 125 | prior := f.read() |
| 126 | marked, ok, err := hostclaim.ReadCompletedRelocation(prior.Claim) |
| 127 | if err != nil || !ok || !marked.Discharged { |
| 128 | t.Fatalf("the completion was not marked: %+v %v %v", marked, ok, err) |
| 129 | } |
| 130 | if row, err := reg.Get(recoveryHost); err != nil || row == nil || row.HoldOp != "" { |
| 131 | t.Fatalf("the hold was not released: %+v %v", row, err) |
| 132 | } |
| 133 | if act.svc.Fenced(recoveryHost) { |
| 134 | t.Fatal("the probe's owned fence was not lifted") |
| 135 | } |
| 136 | if err := hostclaim.StartGuard(prior.Claim, []string{"uk"}, []string{"spa"}, ""); err != nil { |
| 137 | t.Fatalf("the barrier did not open: %v", err) |
| 138 | } |
| 139 | |
| 140 | // 3. The first serving wake, N6 in front as the production |
| 141 | // constructor wires it with moves C1 off. |
| 142 | w := f.controller("boot-1", "controller-1", 4101, "start-1") |
| 143 | socket, posted := handoverRoot(t) |
| 144 | locals := 0 |
| 145 | w.wake.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil } |
| 146 | wireProductionAdmission(w.wake, box, false, claimCrossBoxHandover{Box: box}, nil, |
| 147 | &relocationCompletedHandover{Guard: f.guard, Identity: identity, Socket: socket, Box: box}) |
| 148 | // Prerequisites' predecessor stop: the probe's receipt is found and |
| 149 | // verified; root is asked to stop nothing. |
| 150 | if err := w.wake.stopPredecessor(ctx, prior); err != nil || len(f.requests) != 0 { |
| 151 | t.Fatalf("the predecessor stop: %v (root asked %d stops)", err, len(f.requests)) |
| 152 | } |
| 153 | admitted := w.admit(t, f) |
| 154 | if admitted.Claim.Holder.Session <= 12 || admitted.Claim.Holder.WriterBox != box { |
| 155 | t.Fatalf("not a new writer session on %s: %+v", box, admitted.Claim.Holder) |
| 156 | } |
| 157 | if len(*posted) != 1 || (*posted)[0].Prior != target || (*posted)[0].Next.Session != admitted.Claim.Holder.Session || locals != 0 || len(f.requests) != 0 { |
| 158 | t.Fatalf("root was handed %+v (main's local handover %d, stops %d)", *posted, locals, len(f.requests)) |
| 159 | } |
| 160 | } |
| 161 | |
amadan