amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package release
2
3import (
4 "bytes"
5 "context"
6 "crypto/ed25519"
7 "crypto/rand"
8 "encoding/json"
9 "errors"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "testing"
14 "time"
15
16 "github.com/carlosframework/platform/internal/registry"
17)
18
19type reconstructionCrashStore struct {
20 *healthyRetryStore
21 stage string
22 fail bool
23}
24
25func TestPlan14RootHistoryReconstructionDeliveryPass(t *testing.T) {
26 for _, attack := range []string{"valid", "missing", "forged", "source", "revision"} {
27 t.Run(attack, func(t *testing.T) {
28 h, _, b := rootHistoryFixture(t)
29 r, err := h.read(b)
30 if err != nil {
31 t.Fatal(err)
32 }
33 r.Refusals.Complete = false
34 if err := h.write(r); err != nil {
35 t.Fatal(err)
36 }
37 ctx := t.Context()
38 st := NewDirStore(t.TempDir())
39 priv, pub := testKeys(t)
40 reg := testRegistry(t)
41 addRoute(t, reg, b.Host, b.App, "canary/x", false)
42 shipRelease(t, st, b.Account, b.App, "new", adoptPlatform, []byte("new"))
43 if err := Promote(ctx, st, priv, b.Account, b.App, "canary/x", "new"); err != nil {
44 t.Fatal(err)
45 }
46 status, err := h.ReconstructionStatus()
47 if err != nil {
48 t.Fatal(err)
49 }
50 now := time.Now().UTC()
51 g := HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: status.Identity, Revision: status.Revision, Operation: "delivered", IssuedAt: now, Account: b.Account, App: b.App,
52 Hosts: []HistoryReconstructionHost{{Host: b.Host, Source: "record", Revision: 1}}}
53 if attack == "source" {
54 g.Hosts[0].Source = "forged"
55 }
56 if attack == "revision" {
57 g.Hosts[0].Revision++
58 }
59 signer := RootSigner(priv)
60 if attack == "forged" {
61 _, other, _ := ed25519.GenerateKey(rand.Reader)
62 signer = RootSigner(other)
63 }
64 if attack != "missing" {
65 if err := PublishHistoryReconstruction(ctx, st, signer, g); err != nil {
66 t.Fatal(err)
67 }
68 }
69 deps := &RootHistoryPass{History: h, Resolve: func(context.Context, registry.Route) (HistoryBinding, string, error) { return b, "record", nil }, Installed: func(context.Context, registry.Route, string) (string, error) { return "", nil }}
70 deps.Reconstruct = func(ctx context.Context, b HistoryBinding) error {
71 return h.ReceiveReconstruction(ctx, st, pub, b, now, func(HistoryBinding) (string, error) { return "record", nil })
72 }
73 effects := 0
74 opt := t.TempDir()
75 err = Pass(ctx, reg, st, pub, opt, func(string) error { effects++; return nil }, WithRootHistory(deps))
76 if attack == "valid" {
77 if err != nil || effects != 1 {
78 t.Fatalf("delivered reconstruction: effects=%d err=%v", effects, err)
79 }
80 if live, _ := LiveVersion(opt, b.Host); live != "new" {
81 t.Fatalf("live=%q", live)
82 }
83 if err := deps.Reconstruct(ctx, b); err != nil {
84 t.Fatalf("consumed delivery replay: %v", err)
85 }
86 } else if err == nil || effects != 0 {
87 t.Fatalf("%s: effects=%d err=%v", attack, effects, err)
88 }
89 })
90 }
91}
92
93func (s *reconstructionCrashStore) Write(name string, raw []byte) error {
94 old, _ := s.Read(name)
95 if err := s.healthyRetryStore.Write(name, raw); err != nil {
96 return err
97 }
98 if bytes.Equal(old, raw) {
99 return nil
100 }
101 stage := name
102 if name == reconstructionKey {
103 var state reconstructionState
104 if err := json.Unmarshal(raw, &state); err != nil {
105 return err
106 }
107 if state.Pending != nil {
108 stage = "intent"
109 } else if state.Last != "" {
110 stage = "complete"
111 }
112 }
113 if s.stage == stage {
114 if s.fail {
115 return s.fault
116 }
117 os.Exit(83)
118 }
119 return nil
120}
121
122func reconstructionCrashFixture(t *testing.T) (*RootHistory, *healthyRetryStore, HistoryBinding, ed25519.PublicKey, []byte, time.Time) {
123 t.Helper()
124 h, s, b := rootHistoryFixture(t)
125 status, err := h.ReconstructionStatus()
126 if err != nil {
127 t.Fatal(err)
128 }
129 pub, priv, _ := ed25519.GenerateKey(rand.Reader)
130 now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
131 g := HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: status.Identity, Revision: status.Revision, Operation: "crash", IssuedAt: now, Account: b.Account, App: b.App,
132 Hosts: []HistoryReconstructionHost{{Host: b.Host, Source: "canonical", Revision: 1}}, Holds: map[string]HistoryReconstructionHold{"stable": {Window: time.Hour, SetAt: now}}}
133 wire, err := SignRootEnvelope(RootSigner(priv), g)
134 if err != nil {
135 t.Fatal(err)
136 }
137 return h, s, b, pub, wire, now
138}
139
140func TestPlan14RootHistoryReconstructionCrash(t *testing.T) {
141 resolve := func(HistoryBinding) (string, error) { return "canonical", nil }
142 if dir := os.Getenv("PLAN14_RECONSTRUCTION_CHILD"); dir != "" {
143 pub, err := os.ReadFile(filepath.Join(dir, "pub"))
144 if err != nil {
145 t.Fatal(err)
146 }
147 wire, err := os.ReadFile(filepath.Join(dir, "wire"))
148 if err != nil {
149 t.Fatal(err)
150 }
151 s := &reconstructionCrashStore{healthyRetryStore: &healthyRetryStore{dir: dir}, stage: os.Getenv("PLAN14_RECONSTRUCTION_STAGE")}
152 h := &RootHistory{Store: s}
153 err = h.Reconstruct(pub, wire, "bab", "console", time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC), resolve)
154 t.Fatalf("crash boundary not reached: %v", err)
155 }
156 for _, stage := range []string{"intent", "host", "app", rootBakeKey, "complete"} {
157 t.Run(stage, func(t *testing.T) {
158 h, s, b, pub, wire, now := reconstructionCrashFixture(t)
159 name := stage
160 if stage == "host" {
161 name = historyKey(b)
162 }
163 if stage == "app" {
164 name = historyAppKey(b.Account, b.App)
165 }
166 for key, data := range map[string][]byte{"pub": pub, "wire": wire} {
167 if err := os.WriteFile(filepath.Join(s.dir, key), data, 0600); err != nil {
168 t.Fatal(err)
169 }
170 }
171 cmd := exec.Command(os.Args[0], "-test.run=^TestPlan14RootHistoryReconstructionCrash$")
172 cmd.Env = append(os.Environ(), "PLAN14_RECONSTRUCTION_CHILD="+s.dir, "PLAN14_RECONSTRUCTION_STAGE="+name)
173 out, err := cmd.CombinedOutput()
174 var exit *exec.ExitError
175 if !errors.As(err, &exit) || exit.ExitCode() != 83 {
176 t.Fatalf("child: %v %s", err, out)
177 }
178 var accepted HistoryReconstruction
179 if err := VerifyRootEnvelope(pub, wire, &accepted); err != nil {
180 t.Fatal(err)
181 }
182 acceptedRaw, _ := json.Marshal(accepted)
183 view, err := h.MigrationStatus(b, resolve)
184 if err != nil {
185 t.Fatal(err)
186 }
187 want := ""
188 if stage == "complete" {
189 want = reconstructionDigest(acceptedRaw)
190 }
191 if view.LastReconstructionDigest != want {
192 t.Fatal("crash reported wrong completion")
193 }
194 if stage != "complete" {
195 if _, err := h.Refusals(b); err == nil {
196 t.Fatal("partial reconstruction granted refusal completeness")
197 }
198 if _, err := h.Programs(b); err == nil {
199 t.Fatal("partial reconstruction granted program history")
200 }
201 if _, err := h.AppRefusals(b.Account, b.App); err == nil {
202 t.Fatal("partial reconstruction granted mint inventory")
203 }
204 if _, err := h.LoadBake([]HistoryBinding{b}, now); err == nil {
205 t.Fatal("partial reconstruction granted bake completeness")
206 }
207 if err := h.ResumeReconstruction(func(HistoryBinding) (string, error) { return "changed", nil }); err == nil {
208 t.Fatal("changed canonical source resumed")
209 }
210 }
211 if err := h.ResumeReconstruction(resolve); err != nil {
212 t.Fatal(err)
213 }
214 view, err = h.MigrationStatus(b, resolve)
215 if err != nil || view.LastReconstructionDigest != reconstructionDigest(acceptedRaw) || view.Pending {
216 t.Fatalf("resume completion: %+v %v", view, err)
217 }
218 if _, err := h.AppRefusals(b.Account, b.App); err != nil {
219 t.Fatal(err)
220 }
221 before, _ := s.Read(rootBakeKey)
222 if err := h.ResumeReconstruction(resolve); err != nil {
223 t.Fatal(err)
224 }
225 after, _ := s.Read(rootBakeKey)
226 if !bytes.Equal(before, after) {
227 t.Fatal("resume changed committed clock")
228 }
229 })
230 }
231}
232
233func TestPlan14RootHistoryReconstructionVisibleSync(t *testing.T) {
234 h, s, b, pub, wire, now := reconstructionCrashFixture(t)
235 resolve := func(HistoryBinding) (string, error) { return "canonical", nil }
236 fault := &reconstructionCrashStore{healthyRetryStore: s, stage: historyKey(b), fail: true}
237 h.Store = fault
238 if err := h.Reconstruct(pub, wire, b.Account, b.App, now, resolve); !errors.Is(err, s.fault) {
239 t.Fatalf("write: %v", err)
240 }
241 if _, err := h.Refusals(b); err == nil {
242 t.Fatal("visible unsynced reconstruction granted permission")
243 }
244 fault.stage = ""
245 if err := h.ResumeReconstruction(resolve); err != nil {
246 t.Fatal(err)
247 }
248 if _, err := h.Refusals(b); err != nil {
249 t.Fatal(err)
250 }
251}
252
253func TestPlan14RootHistoryReconstruction(t *testing.T) {
254 for _, attack := range []string{"valid", "signature", "identity", "revision", "source", "expired", "future", "pending", "omitted-deny"} {
255 t.Run(attack, func(t *testing.T) {
256 h, disk, b := rootHistoryFixture(t)
257 now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC)
258 ref := Refusal{Schema: 1, Host: b.Host, App: b.App, Version: "bad", RefusedAt: now, PointerPromotedAt: now}
259 if err := h.AddRefusal(b, ref); err != nil {
260 t.Fatal(err)
261 }
262 status, err := h.ReconstructionStatus()
263 if err != nil {
264 t.Fatal(err)
265 }
266 pub, priv, _ := ed25519.GenerateKey(rand.Reader)
267 g := HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: status.Identity, Revision: status.Revision, Operation: "op1", IssuedAt: now,
268 Account: b.Account, App: b.App, Hosts: []HistoryReconstructionHost{{Host: b.Host, Source: "canonical", Revision: 2, Refusals: []Refusal{ref}}},
269 Holds: map[string]HistoryReconstructionHold{"stable": {Window: time.Hour, SetAt: now}}, Production: true, ProductionSetAt: now}
270 switch attack {
271 case "identity":
272 g.Identity = "other-box"
273 case "revision":
274 g.Revision++
275 case "source":
276 g.Hosts[0].Source = "forged"
277 case "expired":
278 g.IssuedAt = now.Add(-25 * time.Hour)
279 case "future":
280 g.IssuedAt = now.Add(2 * time.Minute)
281 case "pending":
282 if err := h.Begin(b, HistoryAttempt{ID: "attempt", Version: "bad", Source: "canonical", Started: now}); err != nil {
283 t.Fatal(err)
284 }
285 g.Hosts[0].Revision++
286 case "omitted-deny":
287 g.Hosts[0].Refusals = nil
288 }
289 wire, err := SignRootEnvelope(RootSigner(priv), g)
290 if err != nil {
291 t.Fatal(err)
292 }
293 if attack == "signature" {
294 pub, _, _ = ed25519.GenerateKey(rand.Reader)
295 }
296 resolve := func(HistoryBinding) (string, error) { return "canonical", nil }
297 err = h.Reconstruct(pub, wire, b.Account, b.App, now, resolve)
298 if attack != "valid" && attack != "omitted-deny" {
299 if err == nil {
300 t.Fatal("invalid reconstruction accepted")
301 }
302 return
303 }
304 if err != nil {
305 t.Fatal(err)
306 }
307 refs, err := h.AppRefusals(b.Account, b.App)
308 if err != nil || len(refs) != 1 || refs[0].Version != "bad" {
309 t.Fatalf("retained denials: %v %v", refs, err)
310 }
311 clock, err := h.LoadBake([]HistoryBinding{b}, now)
312 if err != nil || clock.Holds[channelClockKey(b.Account, b.App, "stable")].Window != time.Hour {
313 t.Fatalf("bake: %v %v", clock, err)
314 }
315 before, _ := disk.Read(rootBakeKey)
316 if err := h.Reconstruct(pub, wire, b.Account, b.App, now.Add(time.Hour), resolve); err != nil {
317 t.Fatal(err)
318 }
319 after, _ := disk.Read(rootBakeKey)
320 if string(before) != string(after) {
321 t.Fatal("replay changed bake observation")
322 }
323 if err := os.WriteFile(filepath.Join(t.TempDir(), "refusal.json"), []byte(`{"recovered":true}`), 0600); err != nil {
324 t.Fatal(err)
325 }
326 if blocked, err := h.Blocks(b, &ChannelPointer{Version: "bad", PromotedAt: now}); err != nil || !blocked {
327 t.Fatalf("reconstruction cleared deny: %v %v", blocked, err)
328 }
329 })
330 }
331}
332
333func TestPlan14RootHistoryReconstructionUnknownAndFresh(t *testing.T) {
334 h, disk, b := rootHistoryFixture(t)
335 if err := os.Remove(filepath.Join(disk.dir, historyKey(b))); err != nil {
336 t.Fatal(err)
337 }
338 status, err := h.ReconstructionStatus()
339 if err != nil {
340 t.Fatal(err)
341 }
342 if _, err := h.Refusals(b); err == nil {
343 t.Fatal("status initialized history")
344 }
345 pub, priv, _ := ed25519.GenerateKey(rand.Reader)
346 now := time.Now().UTC()
347 g := HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: status.Identity, Revision: status.Revision, Operation: "fresh", IssuedAt: now, Account: b.Account, App: b.App,
348 Hosts: []HistoryReconstructionHost{{Host: b.Host, Source: "canonical"}}}
349 wire, _ := SignRootEnvelope(RootSigner(priv), g)
350 if err := h.Reconstruct(pub, wire, b.Account, b.App, now, func(HistoryBinding) (string, error) { return "canonical", nil }); err != nil {
351 t.Fatal(err)
352 }
353 if _, err := h.Programs(b); err == nil {
354 t.Fatal("reconstruction invented installed programs")
355 }
356 if refs, err := h.Refusals(b); err != nil || len(refs) != 0 {
357 t.Fatalf("explicit empty reconstruction: %v %v", refs, err)
358 }
359}
360
361func TestPlan14RootHistoryObservationKeepsReconstructionRevision(t *testing.T) {
362 h, _, b := rootHistoryFixture(t)
363 before, err := h.read(b)
364 if err != nil {
365 t.Fatal(err)
366 }
367 if err := h.RecordInstalledProgram(b, "old/program"); err != nil {
368 t.Fatal(err)
369 }
370 after, err := h.read(b)
371 if err != nil {
372 t.Fatal(err)
373 }
374 if after.Revision != before.Revision {
375 t.Fatalf("repeat observation invalidated operator reconstruction: %d -> %d", before.Revision, after.Revision)
376 }
377 if err := h.RecordInstalledProgram(b, "new/program"); err != nil {
378 t.Fatal(err)
379 }
380 after, err = h.read(b)
381 if err != nil {
382 t.Fatal(err)
383 }
384 if after.Revision != before.Revision+1 {
385 t.Fatal("new program did not advance revision")
386 }
387}
388
389func TestPlan14RootHistoryReconstructionPreservesOtherRecovery(t *testing.T) {
390 h, _, b, pub, wire, now := reconstructionCrashFixture(t)
391 other := b
392 other.Host = "other.example.com"
393 other.App = "another"
394 if err := h.write(historyRecord{Schema: 1, Binding: other, Revision: 1, Programs: historyPrograms{Revision: 1, Complete: true}, Refusals: historyRefusals{Revision: 1, Complete: true}, Attempt: &HistoryAttempt{ID: "unresolved", Version: "bad", Source: "source", Started: now}}); err != nil {
395 t.Fatal(err)
396 }
397 if err := h.Reconstruct(pub, wire, b.Account, b.App, now, func(HistoryBinding) (string, error) { return "canonical", nil }); err == nil {
398 t.Fatal("reconstruction could block unrelated automatic recovery")
399 }
400 if pending, err := h.Pending(other); err != nil || pending == nil || pending.ID != "unresolved" {
401 t.Fatalf("recovery unavailable: %v %v", pending, err)
402 }
403}
404
405func TestPlan14RootHistoryReconstructionRestartsHistoricalWait(t *testing.T) {
406 h, _, b, pub, wire, now := reconstructionCrashFixture(t)
407 seedRootBake(t, h, b, now.Add(-24*time.Hour))
408 clock, err := h.LoadBake([]HistoryBinding{b}, now.Add(-24*time.Hour))
409 if err != nil {
410 t.Fatal(err)
411 }
412 clock.Observe(b.Account, b.App, "stable", "v1", now.Add(-24*time.Hour))
413 old := now.Add(-24 * time.Hour)
414 clock.Production = map[string]*prodMemory{prodKey(b.Account, b.App): {WasProduction: true, NewestSetAt: old, OffSetAt: old, OffFirstSeen: old}}
415 clock.Holds = map[string]*holdMemory{channelClockKey(b.Account, b.App, "stable"): {Window: 2 * time.Hour, Pending: time.Minute, PendingAt: old, DeletedAt: old}}
416 clock.Channels[channelClockKey(b.Account, "other", "stable")] = map[string]time.Time{"v1": old}
417 if err := clock.Save(); err != nil {
418 t.Fatal(err)
419 }
420 var grant HistoryReconstruction
421 if err := VerifyRootEnvelope(pub, wire, &grant); err != nil {
422 t.Fatal(err)
423 }
424 grant.BakeRevision = 2
425 priv, pub := testKeys(t)
426 wire, err = SignRootEnvelope(RootSigner(priv), grant)
427 if err != nil {
428 t.Fatal(err)
429 }
430 if err := h.Reconstruct(pub, wire, b.Account, b.App, now, func(HistoryBinding) (string, error) { return "canonical", nil }); err != nil {
431 t.Fatal(err)
432 }
433 clock, err = h.LoadBake([]HistoryBinding{b}, now)
434 if err != nil {
435 t.Fatal(err)
436 }
437 eligible, ok := clock.EligibleAt(b.Account, b.App, "stable", "v1", map[string]time.Duration{"stable": time.Hour})
438 if !ok || eligible.Before(now.Add(time.Hour)) {
439 t.Fatalf("reconstructed bound inherited elapsed time: %v %v", eligible, ok)
440 }
441 mem := clock.Holds[channelClockKey(b.Account, b.App, "stable")]
442 if mem.Window != 2*time.Hour || !mem.PendingAt.Equal(now) || !mem.DeletedAt.Equal(now) {
443 t.Fatalf("historical hold changed: %+v", mem)
444 }
445 prod := clock.Production[prodKey(b.Account, b.App)]
446 if !prod.WasProduction || !prod.OffFirstSeen.Equal(now) {
447 t.Fatalf("historical production changed: %+v", prod)
448 }
449 if !clock.Channels[channelClockKey(b.Account, "other", "stable")]["v1"].Equal(old) {
450 t.Fatal("reconstruction changed another app's clock")
451 }
452}
453