amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "encoding/json"
6 "github.com/carlosframework/platform/internal/hostactors"
7 "github.com/carlosframework/platform/internal/hostfloor"
8 "github.com/carlosframework/platform/internal/release"
9 "os"
10 "path/filepath"
11 "strings"
12 "testing"
13)
14
15func TestFloorRefusalKeepsRecoveryJournalAndExecutesNothing(t *testing.T) {
16 for _, phase := range []string{systemPhaseBackingUp, systemPhaseInstalled, systemPhaseGated} {
17 t.Run(phase, func(t *testing.T) {
18 f := newApplyFixture(t, "stable", "old-build")
19 f.a.floor = &hostfloor.Disk{Dir: t.TempDir(), Bucket: "bucket", PublicKey: f.pub, Participation: f.a.participation}
20 _ = os.MkdirAll(f.a.rootDir(), 0700)
21 if err := f.a.journal(systemJournal{Phase: phase, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil {
22 t.Fatal(err)
23 }
24 if err := f.a.Run(context.Background(), true); err == nil {
25 t.Fatal("missing policy accepted")
26 }
27 if _, err := os.Stat(filepath.Join(f.a.rootDir(), "journal.json")); err != nil {
28 t.Fatal("journal discarded", err)
29 }
30 if len(f.calls) != 0 {
31 t.Fatal("ran before guard", f.calls)
32 }
33 })
34 }
35}
36
37func TestUnpublishedCompletionAndRecovery(t *testing.T) {
38 f := newApplyFixture(t, "stable", "old-build")
39 f.a.floor = &hostfloor.Disk{Dir: t.TempDir(), Bucket: "bucket", PublicKey: f.pub, Participation: f.a.participation}
40 raw, _ := json.Marshal(hostfloor.Policy{Schema: 1, DeploymentBucket: "bucket", Mode: "unpublished"})
41 if err := hostfloor.Durable(filepath.Join(f.a.floor.Dir, "policy.json"), raw, 0644); err != nil {
42 t.Fatal(err)
43 }
44 if err := f.a.beginGuard(t.Context(), nil); err != nil {
45 t.Fatal(err)
46 }
47 if err := f.a.endGuard(t.Context()); err != nil {
48 t.Fatal(err)
49 }
50 if err := f.a.checkGuardRecovery(t.Context()); err != nil {
51 t.Fatal(err)
52 }
53 if _, err := os.Stat(filepath.Join(f.a.floor.Dir, "activation.json")); !os.IsNotExist(err) {
54 t.Fatal(err)
55 }
56}
57
58func guardRecoveryFixture(t *testing.T) (*applyFixture, string) {
59 t.Helper()
60 f := newApplyFixture(t, "stable", "old-build")
61 a := f.a
62 a.floor = &hostfloor.Disk{Dir: t.TempDir(), Bucket: "bucket", PublicKey: f.pub, Participation: f.a.participation}
63 a.guardOpen = os.Open // Isolate the state machine from root-owned path checks.
64 sg := release.RootSigner(f.priv)
65 floor, _ := json.Marshal(hostfloor.Floor{Min: 1, MinRelease: "v1.2.3"})
66 auth, err := release.SignRootEnvelope(sg, hostfloor.Authorization{Schema: 1, Purpose: "moves-protocol", DeploymentBucket: "bucket", Key: hostfloor.Key, SHA256: hostfloor.Digest(floor)})
67 if err != nil {
68 t.Fatal(err)
69 }
70 policy, _ := json.Marshal(hostfloor.Policy{Schema: 1, DeploymentBucket: "bucket", Mode: "armed", Floor: floor, Authorization: auth})
71 if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "policy.json"), policy, 0644); err != nil {
72 t.Fatal(err)
73 }
74 body, err := os.ReadFile(systemBinaryPath(a.optDir))
75 if err != nil {
76 t.Fatal(err)
77 }
78 hash := hostfloor.Digest(body)
79 rel := release.SystemRelease{Version: "old-build", ReleaseTag: "v1.2.3", MovesProtocol: 1, Artifacts: map[string]release.SystemArtifact{a.platform: {Key: release.SystemArtifactKey("old-build", a.platform), SHA256: hash}}}
80 if _, err := release.PromoteSystem(t.Context(), f.store, sg, "router", a.channel, rel, release.SystemPromoteOptions{}); err != nil {
81 t.Fatal(err)
82 }
83 identity, _, err := f.store.Get(t.Context(), release.SystemPointerKey("router", a.channel))
84 if err != nil {
85 t.Fatal(err)
86 }
87 if err := os.Mkdir(filepath.Join(a.floor.Dir, "identities"), 0755); err != nil {
88 t.Fatal(err)
89 }
90 if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "identities", "v1.2.3.json"), identity, 0644); err != nil {
91 t.Fatal(err)
92 }
93 id := strings.Repeat("a", 32)
94 enrollment := strings.Repeat("b", 32)
95 box := hostactors.Box{Enrollment: enrollment, Namespace: "test", BoxID: "box", Label: "box", Region: "region", Pool: "pool", Cloud: hostactors.Cloud{Provider: "aws", Account: "account", Region: "region", InstanceID: "instance"}, EvidenceSHA256: strings.Repeat("c", 64), State: "active"}
96 h := hostactors.Reservation{ID: id, FloorSHA256: hostfloor.Digest(floor), Actors: map[string]string{"box:box": enrollment}, Releases: map[string]hostactors.Report{"box:box": {Enrollment: enrollment}}, Hashes: map[string][]string{"box:box": {hash, hash, hash}}}
97 roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Boxes: map[string]hostactors.Box{"box": box}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{id: h}}
98 if err := roster.Validate("bucket"); err != nil {
99 t.Fatal(err)
100 }
101 wire, err := release.SignRootEnvelope(sg, roster)
102 if err != nil {
103 t.Fatal(err)
104 }
105 tx, _ := json.Marshal(guardTransaction{Roster: wire, ID: id, Actor: "box:box", Enrollment: enrollment})
106 if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "activation.json"), tx, 0644); err != nil {
107 t.Fatal(err)
108 }
109 if err := os.MkdirAll(filepath.Join(a.dataDir, ".hostexec"), 0755); err != nil {
110 t.Fatal(err)
111 }
112 return f, id
113}
114func TestGuardRecoveryEveryTerminalCrashState(t *testing.T) {
115 for _, state := range []string{"prepared", "staging-failed", "journal-removed", "completion-written", "marker-removed"} {
116 t.Run(state, func(t *testing.T) {
117 f, id := guardRecoveryFixture(t)
118 a := f.a
119 marker := filepath.Join(a.floor.Dir, "update-begun.json")
120 completed := filepath.Join(a.floor.Dir, "completed.json")
121 if state != "prepared" {
122 if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
123 t.Fatal(err)
124 }
125 }
126 if state == "journal-removed" {
127 if err := a.journal(systemJournal{Phase: systemPhaseGated, Target: "old-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil {
128 t.Fatal(err)
129 }
130 if err := hostfloor.RemoveDurable(a.file("journal.json")); err != nil {
131 t.Fatal(err)
132 }
133 }
134 if state == "completion-written" || state == "marker-removed" {
135 if err := hostfloor.Durable(completed, []byte(id), 0644); err != nil {
136 t.Fatal(err)
137 }
138 }
139 if state == "marker-removed" {
140 if err := hostfloor.RemoveDurable(marker); err != nil {
141 t.Fatal(err)
142 }
143 }
144 if err := a.Run(t.Context(), true); err != nil {
145 t.Fatal(err)
146 }
147 if err := a.Run(t.Context(), true); err != nil {
148 t.Fatal("replay", err)
149 }
150 raw, err := os.ReadFile(completed)
151 if err != nil || string(raw) != id {
152 t.Fatal("completion absent", err, string(raw))
153 }
154 if _, err := os.Stat(marker); !os.IsNotExist(err) {
155 t.Fatal("marker stranded", err)
156 }
157 if len(f.calls) != 0 {
158 t.Fatal("recovery executed a binary", f.calls)
159 }
160 if err := a.floor.Begin(t.Context(), "next", a.dataDir); err != nil {
161 t.Fatal("subsequent update blocked", err)
162 }
163 })
164 }
165}
166func TestGuardRecoveryRetainsMarkerOnInvalidImageOrMarker(t *testing.T) {
167 for _, bad := range []string{"image", "marker", "completion-io"} {
168 t.Run(bad, func(t *testing.T) {
169 f, id := guardRecoveryFixture(t)
170 a := f.a
171 if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
172 t.Fatal(err)
173 }
174 marker := filepath.Join(a.floor.Dir, "update-begun.json")
175 switch bad {
176 case "image":
177 if err := os.WriteFile(systemBinaryPath(a.optDir), []byte("wrong executable"), 0755); err != nil {
178 t.Fatal(err)
179 }
180 case "marker":
181 if err := os.WriteFile(marker, []byte(`{"id":"other"}`), 0644); err != nil {
182 t.Fatal(err)
183 }
184 case "completion-io":
185 if err := os.Mkdir(filepath.Join(a.floor.Dir, "completed.json"), 0755); err != nil {
186 t.Fatal(err)
187 }
188 }
189 if err := a.Run(t.Context(), true); err == nil {
190 t.Fatal("unsafe recovery succeeded")
191 }
192 if _, err := os.Stat(marker); err != nil {
193 t.Fatal("denial erased", err)
194 }
195 if bad == "completion-io" {
196 if err := os.Remove(filepath.Join(a.floor.Dir, "completed.json")); err != nil {
197 t.Fatal(err)
198 }
199 if err := a.Run(t.Context(), true); err != nil {
200 t.Fatal("retry failed", err)
201 }
202 }
203 })
204 }
205}
206
207func TestGuardRecoveryRefusesControllersAfterBegin(t *testing.T) {
208 t.Parallel()
209 states := []struct {
210 name string
211 malformed bool
212 unreadable bool
213 }{
214 {name: "pending"},
215 {name: "active"},
216 {name: "lost"},
217 {name: "malformed", malformed: true},
218 {name: "unreadable", unreadable: true},
219 }
220 paths := []struct {
221 name string
222 journal bool
223 marker bool
224 }{
225 {name: "journal-backing-up", journal: true, marker: true},
226 {name: "terminal-marker", marker: true},
227 {name: "terminal-no-marker"},
228 }
229 for _, state := range states {
230 for _, path := range paths {
231 t.Run(state.name+"-"+path.name, func(t *testing.T) {
232 f, id := guardRecoveryFixture(t)
233 a := f.a
234 if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
235 t.Fatal(err)
236 }
237 marker := filepath.Join(a.floor.Dir, "update-begun.json")
238 binary := systemBinaryPath(a.optDir)
239 before, err := os.ReadFile(binary)
240 if err != nil {
241 t.Fatal(err)
242 }
243 if path.journal {
244 if err := a.journal(systemJournal{Phase: systemPhaseBackingUp, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil {
245 t.Fatal(err)
246 }
247 }
248 controllers := filepath.Join(a.dataDir, ".hostexec")
249 if state.unreadable {
250 if err := os.RemoveAll(controllers); err != nil {
251 t.Fatal(err)
252 }
253 if err := os.WriteFile(controllers, []byte("not a directory"), 0644); err != nil {
254 t.Fatal(err)
255 }
256 } else if state.malformed {
257 if err := os.WriteFile(filepath.Join(controllers, "host.json"), []byte("{"), 0600); err != nil {
258 t.Fatal(err)
259 }
260 } else {
261 raw, err := json.Marshal(map[string]string{"state": state.name})
262 if err != nil {
263 t.Fatal(err)
264 }
265 if err := os.WriteFile(filepath.Join(controllers, "host.json"), raw, 0600); err != nil {
266 t.Fatal(err)
267 }
268 }
269 if !path.marker {
270 if err := hostfloor.RemoveDurable(marker); err != nil {
271 t.Fatal(err)
272 }
273 }
274 if err := a.Run(t.Context(), true); err == nil {
275 t.Fatal("unsafe recovery succeeded")
276 }
277 if path.marker {
278 if _, err := os.Stat(marker); err != nil {
279 t.Fatal("marker discarded", err)
280 }
281 } else if _, err := os.Stat(marker); !os.IsNotExist(err) {
282 t.Fatal("marker unexpectedly present", err)
283 }
284 if path.journal {
285 if _, err := os.Stat(a.file("journal.json")); err != nil {
286 t.Fatal("journal discarded", err)
287 }
288 }
289 if _, err := os.Stat(filepath.Join(a.floor.Dir, "completed.json")); !os.IsNotExist(err) {
290 t.Fatal("completion written", err)
291 }
292 after, err := os.ReadFile(binary)
293 if err != nil || string(after) != string(before) {
294 t.Fatal("binary changed", err)
295 }
296 if len(f.calls) != 0 {
297 t.Fatal("recovery executed a binary", f.calls)
298 }
299 })
300 }
301 }
302}
303
304func TestGuardRejectsOldPointerForUnpublishedActivation(t *testing.T) {
305 f, id := guardRecoveryFixture(t)
306 a := f.a
307 ctx := t.Context()
308 sg := release.RootSigner(f.priv)
309 old, _, err := release.GetSystemPointer(ctx, f.store, f.pub, "router", a.channel)
310 if err != nil {
311 t.Fatal(err)
312 }
313 tx, err := a.guardTransaction()
314 if err != nil {
315 t.Fatal(err)
316 }
317 var roster hostactors.Roster
318 if err := release.VerifyRootEnvelope(f.pub, tx.Roster, &roster); err != nil {
319 t.Fatal(err)
320 }
321 publication, err := release.PrepareSystemPromotion(ctx, f.store, sg, "router", a.channel, old.Current, release.SystemPromoteOptions{}, id)
322 if err != nil {
323 t.Fatal(err)
324 }
325 h := roster.Activations[id]
326 h.Publication = &publication
327 h.RequestSHA256 = strings.Repeat("d", 64)
328 h.UpdateMode = "promote"
329 roster.Activations[id] = h
330 raw, err := release.SignRootEnvelope(sg, roster)
331 if err != nil {
332 t.Fatal(err)
333 }
334 if _, err := f.store.Put(ctx, hostactors.Key, raw, ""); err != nil {
335 t.Fatal(err)
336 }
337 identity, _, err := f.store.Get(ctx, release.SystemPointerKey("router", a.channel))
338 if err != nil {
339 t.Fatal(err)
340 }
341 if err := release.PublishSystemIdentity(ctx, f.store, f.pub, identity); err != nil {
342 t.Fatal(err)
343 }
344 if err := os.MkdirAll(filepath.Join(a.etcDir, "carlos"), 0755); err != nil {
345 t.Fatal(err)
346 }
347 enrollment, _ := json.Marshal(hostfloor.Enrollment{Box: "box", Enrollment: tx.Enrollment, InstanceID: "instance"})
348 if err := hostfloor.Durable(filepath.Join(a.etcDir, "carlos", "host-actor.json"), enrollment, 0644); err != nil {
349 t.Fatal(err)
350 }
351 if err := hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "activation.json")); err != nil {
352 t.Fatal(err)
353 }
354 if err := a.prepareGuard(ctx, old); err == nil {
355 t.Fatal("old same-target pointer consumed unpublished activation")
356 }
357 if _, err := os.Stat(filepath.Join(a.floor.Dir, "activation.json")); !os.IsNotExist(err) {
358 t.Fatalf("refusal wrote transaction: %v", err)
359 }
360 current, err := release.PublishSystemPublication(ctx, f.store, f.pub, publication)
361 if err != nil {
362 t.Fatal(err)
363 }
364 if err := a.prepareGuard(ctx, current); err != nil {
365 t.Fatal("published activation refused", err)
366 }
367}
368
369// M8: a dead pending controller is recovered by root in the no-journal
370// branches too, not only while a journal is open. Recovery that leaves the
371// controller in place still refuses.
372func TestPlan9M8GuardRecoveryRecoversPendingWithoutJournal(t *testing.T) {
373 for _, path := range []struct {
374 name string
375 marker bool
376 }{{"terminal-marker", true}, {"terminal-no-marker", false}} {
377 for _, recovers := range []bool{true, false} {
378 name := path.name + "-recovered"
379 if !recovers {
380 name = path.name + "-still-live"
381 }
382 t.Run(name, func(t *testing.T) {
383 f, id := guardRecoveryFixture(t)
384 a := f.a
385 if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil {
386 t.Fatal(err)
387 }
388 marker := filepath.Join(a.floor.Dir, "update-begun.json")
389 state := filepath.Join(a.dataDir, ".hostexec", "host.json")
390 if err := os.WriteFile(state, []byte(`{"state":"pending"}`), 0600); err != nil {
391 t.Fatal(err)
392 }
393 if !path.marker {
394 if err := hostfloor.RemoveDurable(marker); err != nil {
395 t.Fatal(err)
396 }
397 }
398 calls := 0
399 old := recoverPendingControllers
400 t.Cleanup(func() { recoverPendingControllers = old })
401 recoverPendingControllers = func(_ context.Context, dataDir, guardDir string) error {
402 calls++
403 if dataDir != a.dataDir || guardDir != a.floor.Dir {
404 t.Errorf("recovery dirs %s %s", dataDir, guardDir)
405 }
406 if recovers {
407 return os.Remove(state)
408 }
409 return nil
410 }
411 err := a.Run(t.Context(), true)
412 if calls == 0 {
413 t.Fatal("root never tried to recover the pending controller")
414 }
415 completed := filepath.Join(a.floor.Dir, "completed.json")
416 if !recovers {
417 if err == nil {
418 t.Fatal("a controller that survived recovery was settled")
419 }
420 if _, statErr := os.Stat(completed); !os.IsNotExist(statErr) {
421 t.Fatal("completion written", statErr)
422 }
423 return
424 }
425 if err != nil {
426 t.Fatal(err)
427 }
428 if raw, err := os.ReadFile(completed); err != nil || string(raw) != id {
429 t.Fatal("completion absent", err, string(raw))
430 }
431 if len(f.calls) != 0 {
432 t.Fatal("recovery executed a binary", f.calls)
433 }
434 })
435 }
436 }
437}
438