amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "crypto/ed25519"
6 "crypto/sha256"
7 "encoding/hex"
8 "encoding/json"
9 "fmt"
10 "github.com/carlosframework/platform/internal/activator"
11 "github.com/carlosframework/platform/internal/hostbox"
12 "github.com/carlosframework/platform/internal/hostclaim"
13 "github.com/carlosframework/platform/internal/moves"
14 "github.com/carlosframework/platform/internal/provision"
15 "github.com/carlosframework/platform/internal/registry"
16 "github.com/carlosframework/platform/internal/release"
17 "github.com/carlosframework/platform/internal/termfence"
18 "os/exec"
19 "path/filepath"
20 "strings"
21 "testing"
22 "time"
23)
24
25type plan12FenceAuthority struct {
26 reject bool
27 home string
28}
29
30func (a plan12FenceAuthority) ValidateFence(context.Context, termfence.Record) (hostclaim.FenceBinding, error) {
31 if a.reject {
32 return hostclaim.FenceBinding{}, hostclaim.ErrFence
33 }
34 return hostclaim.FenceBinding{FleetHomeRegion: a.home, Region: "uk", RosterRevision: 1, RosterETag: "roster"}, nil
35}
36func plan12Handover(t *testing.T, kind, mode string) (*recoveryFixture, hostclaim.Observed, claimCrossBoxHandover) {
37 t.Helper()
38 f := newRecoveryFixture(t)
39 o := f.read()
40 key := moves.StoppedKey(moves.ReportPrefix("bab", "jam", o.Host, "m1"), "target")
41 journal := map[string]string{hostclaim.JournalEvidenceTarget: key, hostclaim.JournalTargetSession: "2", hostclaim.JournalTargetGeneration: "target-gen"}
42 if mode == "wrong-session" {
43 journal[hostclaim.JournalTargetSession] = "3"
44 }
45 pub := f.signer.Public().(ed25519.PublicKey)
46 h := claimCrossBoxHandover{Guard: f.guard, Evidence: f.claims, Box: "box-1", PublicKey: pub, FenceAuthority: plan12FenceAuthority{reject: mode == "unverified-fence"}}
47 if mode == "fence" || mode == "unverified-fence" {
48 record := termfence.Record{V: 1, Box: "box-2", Enrollment: strings.Repeat("a", 32), State: "terminated", ObservedAt: time.Now().UTC().Format(time.RFC3339Nano)}
49 raw, err := json.Marshal(record)
50 if err != nil {
51 t.Fatal(err)
52 }
53 sum := sha256.Sum256(raw)
54 ref := hostclaim.FenceRef{Enrollment: record.Enrollment, Key: "control/box-fences/" + record.Enrollment + ".json", SHA256: hex.EncodeToString(sum[:])}
55 if _, err = f.claims.Put(t.Context(), ref.Key, raw, ""); err != nil {
56 t.Fatal(err)
57 }
58 refRaw, _ := json.Marshal(ref)
59 journal = map[string]string{"fence.box-2": string(refRaw)}
60 }
61 if mode == "loss" || mode == "wrong-loss-op" {
62 journal = map[string]string{"loss.loss_1": `{"kind":"move-target-lost","target":"box-2"}`}
63 }
64 var err error
65 o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {
66 c.Op = plan10J2Op(kind, hostclaim.PhaseProbing)
67 c.Op.Journal = journal
68 c.Op.Intent["target_enrollment"] = strings.Repeat("a", 32)
69 c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = 2, "target-gen", "box-2"
70 if mode == "loss" || mode == "wrong-loss-op" {
71 c.LossRefs = []string{"loss_1"}
72 }
73 return nil
74 })
75 if err != nil {
76 t.Fatal(err)
77 }
78 if err = f.guard.Activate(t.Context(), o.Host, "box-1", nil); err != nil {
79 t.Fatal(err)
80 }
81 report := plan12Stopped(o, "target", "box-2")
82 if mode == "wrong-op" {
83 report.OpID = "another"
84 }
85 raw, err := json.Marshal(report)
86 if err != nil {
87 t.Fatal(err)
88 }
89 if _, err = f.claims.Put(t.Context(), key, raw, ""); err != nil {
90 t.Fatal(err)
91 }
92
93 if mode == "loss" || mode == "wrong-loss-op" {
94 now := time.Now().UTC()
95 r := hostclaim.LossRecord{V: 1, ID: "loss_1", State: "prepared", Actor: "operator", AcceptedAt: now, Host: o.Host, HolderIncarnation: o.Claim.Holder.Incarnation, HolderStamp: o.Claim.Holder.Stamp, HolderSession: 2, ExpectedClaimGen: o.Claim.Gen, ExpectedClaimETag: o.ETag, OpID: "m1", OpKind: hostclaim.KindLanding, OpPhase: "probing", Box: "box-2", Enrollment: "enroll", QuarantineRevision: 1,
96 CloudObservation: hostclaim.LossCloudObservation{ObservedAt: now, Account: "123", Region: "eu-west-1", InstanceID: "i-1", State: "stopped", RequestID: "request"}, OperatorAttestation: hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true}, FencedAt: now,
97 ConfirmedConsequences: hostclaim.LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(`{"kind":"move-target-lost"}`)}, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}, WriteSources: []hostclaim.WriteSource{{Reason: "target", Box: "box-2", Session: 2, Generation: "target-gen", Region: "uk", Baseline: "target-snapshot"}}}
98 if mode == "wrong-loss-op" {
99 r.OpID = "other"
100 }
101 made, err := hostclaim.CreateLossRecord(t.Context(), f.claims, release.RootSigner(f.signer), r)
102 if err != nil {
103 t.Fatal(err)
104 }
105 r.State, r.ResultClaimGen = "committed", o.Claim.Gen
106 if _, err = hostclaim.ReplaceLossRecord(t.Context(), f.claims, release.RootSigner(f.signer), made, r); err != nil {
107 t.Fatal(err)
108 }
109 }
110 h.Identity = handoverRecoveryIdentity(t, hostbox.Actor{Box: "box-1", Enrollment: "enrollment-1", Revision: 7, Binding: "7", Region: "ie"})
111 h.Inventory, err = openLegacyInventory(t, filepath.Join(t.TempDir(), "inventory.db"))
112 if err != nil {
113 t.Fatal(err)
114 }
115 t.Cleanup(func() { h.Inventory.Close() })
116 h.Registry, err = registry.Open(filepath.Join(t.TempDir(), "registry.db"))
117 if err != nil {
118 t.Fatal(err)
119 }
120 t.Cleanup(func() { h.Registry.Close() })
121 row := registry.Route{Host: o.Host, Kind: "instance", DBPath: provision.NestedDBPath(f.guard.DataDir, o.Host), ClaimOwnership: registry.ClaimOwnership{ClaimIncarnation: o.Claim.Holder.Incarnation, ClaimStamp: o.Claim.Holder.Stamp, ClaimOwner: "old-owner", ClaimRowID: "old-row"}}
122 if err = h.Registry.Upsert(row); err != nil {
123 t.Fatal(err)
124 }
125 if err = h.Inventory.WriteStopInventory(t.Context(), hostbox.StopInventory{Complete: true, Host: o.Host, Incarnation: row.ClaimIncarnation, InstallationStamp: row.ClaimStamp, Box: "box-1", Enrollment: "enrollment-1", DirectoryRevision: "7", Entries: []hostbox.StopInventoryEntry{{Kind: "route", ArtifactID: "route", ManagedRoot: "registry", RelativePath: o.Host, RowIdentity: "old-row", Ownership: "old-owner", Materialization: "materialized"}}}); err != nil {
126 t.Fatal(err)
127 }
128 o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {
129 c.Op.Journal[hostclaim.JournalSourceStamp] = fmt.Sprint(row.ClaimStamp)
130 return nil
131 })
132 if err != nil {
133 t.Fatal(err)
134 }
135 return f, o, h
136}
137func TestPlan12RecoveryPredecessorEvidence(t *testing.T) {
138 t.Parallel()
139 for _, kind := range []string{hostclaim.KindRestoring} {
140 for _, mode := range []string{"stop", "loss", "fence", "wrong-session", "wrong-op", "wrong-loss-op", "unverified-fence"} {
141 t.Run(kind+"/"+mode, func(t *testing.T) {
142 _, o, h := plan12Handover(t, kind, mode)
143 if !claimRemotePredecessor(o, "box-1") {
144 t.Fatal("local path selected")
145 }
146 err := h.Before(t.Context(), o)
147 valid := mode == "stop" || mode == "loss" || mode == "fence"
148 if valid && err != nil || !valid && err == nil {
149 t.Fatalf("%s: %v", mode, err)
150 }
151 })
152 }
153 }
154}
155func TestPlan12RecoveryAdmissionAfterTargetSession(t *testing.T) {
156 t.Parallel()
157 for _, kind := range []string{hostclaim.KindRestoring} {
158 t.Run(kind, func(t *testing.T) {
159 f, o, h := plan12Handover(t, kind, "stop")
160 c := f.controller("boot-1", "controller-1", 4101, "start-1")
161 c.wake.Inputs = plan9D2Inputs{recoveryInputs{f}}
162 c.wake.Operations = &plan9D2Attempts{}
163 c.wake.ProbeReady = func(context.Context, *exec.Cmd, registry.Route, ClaimLaunchInputs, string) error { return nil }
164 wireClaimAdmission(c.wake, "box-1", func(context.Context, hostclaim.Observed, hostclaim.Observed) error {
165 t.Fatal("local predecessor handover")
166 return nil
167 }, h, nil, nil)
168 if err := c.wake.stopPredecessor(t.Context(), o); err != nil {
169 t.Fatal(err)
170 }
171 live, cancel := context.WithCancel(t.Context())
172 defer cancel()
173 done := make(chan struct{})
174 go func() { <-live.Done(); close(done) }()
175 c.wake.mu.Lock()
176 c.wake.pending[o.Host] = &claimWakeAttempt{observed: o, held: recoveryHeld{}, ctx: live, cancel: cancel, done: done}
177 c.wake.mu.Unlock()
178 got, err := c.runtime.Admit(t.Context(), registry.Route{Host: o.Host}, o, activator.StartPolicy{Probe: true})
179 if err != nil {
180 t.Fatal(err)
181 }
182 if got.Claim.Holder.Session <= 2 || got.Claim.Holder.Stamp != got.Claim.Holder.Session || got.Claim.Holder.WriterBox != "box-1" {
183 t.Fatal(got)
184 }
185 })
186 }
187}
188
189func plan12Stopped(o hostclaim.Observed, role, box string) moves.StoppedReport {
190 return moves.StoppedReport{V: 1, Role: role, Box: box, Host: o.Host, OpID: o.Claim.Op.ID, Incarnation: o.Claim.Holder.Incarnation, ClaimGen: o.Claim.Gen, CheckedAt: time.Now().UTC(), Steps: map[string]moves.Check{moves.StepTree: {OK: true}, moves.StepStray: {OK: true}, moves.StepFDs: {OK: true}, moves.StepLease: {OK: true}, moves.StepReplica: {OK: true}}}
191}
192func TestPlan12RecoveryLocalRouting(t *testing.T) {
193 for _, kind := range []string{hostclaim.KindRestoring, hostclaim.KindReverting} {
194 for _, box := range []string{"", "box-1"} {
195 o := plan10J2Observed(kind, hostclaim.PhaseProbing, "box-1")
196 o.Claim.Holder = &hostclaim.Holder{WriterBox: box}
197 if claimRemotePredecessor(o, "box-1") {
198 t.Fatal(kind, box)
199 }
200 called := false
201 hook := claimAdmissionHandover("box-1", func(context.Context, hostclaim.Observed, hostclaim.Observed) error { called = true; return nil }, func(context.Context, hostclaim.Observed, hostclaim.Observed) error { t.Fatal("remote"); return nil })
202 if err := hook(t.Context(), o, o); err != nil || !called {
203 t.Fatal(err)
204 }
205 }
206 }
207}
208
209func TestPlan12ConsumedFenceSurvivesRosterChange(t *testing.T) {
210 t.Parallel()
211 f, o, h := plan12Handover(t, hostclaim.KindRestoring, "fence")
212 rb := "rb_abcdefghijklmnopqrstuvwxyz"
213 var ref hostclaim.FenceRef
214 if err := json.Unmarshal([]byte(o.Claim.Op.Journal["fence.box-2"]), &ref); err != nil {
215 t.Fatal(err)
216 }
217 o, err := hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {
218 c.Op.Exec = "box-2"
219 c.Op.Phase = hostclaim.PhaseReleasing
220 c.Op.Intent["rollback"] = rb
221 return nil
222 })
223 if err != nil {
224 t.Fatal(err)
225 }
226 fence, err := hostclaim.VerifyMoveTargetFence(t.Context(), f.claims, o, ref, h.FenceAuthority)
227 if err != nil {
228 t.Fatal(err)
229 }
230
231 // Admission consumes a previously grant-released claim. The grant itself
232 // is exercised in hostops.TestPlan12RecoveryFenceGrant.
233 o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error {
234 raw, _ := json.Marshal(ref)
235 sum := sha256.Sum256([]byte(c.Op.ID + "\x00box-2\x00" + ref.Enrollment + "\x00" + ref.SHA256))
236 id := "auto-" + hex.EncodeToString(sum[:16])
237 loss := hostclaim.AutomaticLoss{ID: id, Box: "box-2", Enrollment: ref.Enrollment, Fence: ref, ObservedAt: fence.ObservedAt().Format(time.RFC3339Nano), Op: c.Op.ID, Kind: c.Op.Kind, Phase: c.Op.Phase, Role: "target", Incarnation: c.Holder.Incarnation, Stamp: c.Holder.Stamp, Session: c.Holder.Session, Generation: c.Holder.Generation, Timers: hostclaim.TimersUnknown}
238 if len(c.Holder.Admissions) > 0 {
239 a := c.Holder.Admissions[len(c.Holder.Admissions)-1]
240 loss.LastAdmission = &a
241 }
242 c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseReleased, "none"
243 c.Op.Intent["rollback"] = rb
244 c.Op.Participants = []string{c.Op.Intent["source"]}
245 c.Op.Journal["fence.box-2"], c.Op.Journal["fenced_target_released"] = string(raw), string(raw)
246 c.Lost, c.LossRefs, c.EvidenceRefs = []string{"box-2"}, []string{id}, []hostclaim.AutomaticLoss{loss}
247 return nil
248 })
249 if err != nil {
250 t.Fatal(err)
251 }
252 o, err = hostclaim.Update(t.Context(), f.claims, o, func(c *hostclaim.Claim) error { c.Op.Exec = "box-1"; c.Op.Phase = hostclaim.PhaseProbing; return nil })
253 if err != nil {
254 t.Fatal(err)
255 }
256 h.FenceAuthority = plan12FenceAuthority{reject: true}
257 if err := h.Before(t.Context(), o); err != nil {
258 t.Fatal("consumed evidence depended on roster", err)
259 }
260 o.Claim.EvidenceRefs = nil
261 if err := h.Before(t.Context(), o); err == nil {
262 t.Fatal("journal alone accepted as durable loss")
263 }
264}
265