amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "crypto/ed25519"
5 "encoding/json"
6 "os"
7 "strings"
8 "testing"
9 "time"
10
11 "github.com/carlosframework/platform/internal/historyreadback"
12 "github.com/carlosframework/platform/internal/hostactors"
13 "github.com/carlosframework/platform/internal/hostfloor"
14 "github.com/carlosframework/platform/internal/release"
15 "github.com/carlosframework/platform/internal/rootstate"
16)
17
18func TestPlan14RecoveryRetirementProductionCallback(t *testing.T) {
19 for _, replacement := range []bool{false, true} {
20 t.Run(map[bool]string{false: "reconstruct", true: "replace"}[replacement], func(t *testing.T) {
21 e, u, c, source, pub := signedRecoveryFixture(t)
22 rt, err := e.root.reg.Get(observedHost)
23 if err != nil {
24 t.Fatal(err)
25 }
26 h := seedHostRootHistory(t, *rt)
27 n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink}
28 if _, err := n.recover(t.Context(), rt.Host, c, func(string) string { return "5s" }, e.root.reg); err != nil {
29 t.Fatal(err)
30 }
31 disk, err := c.attempts()
32 if err != nil {
33 t.Fatal(err)
34 }
35 good, err := readRollbackAttempt(disk, rt.Host)
36 if err != nil {
37 t.Fatal(err)
38 }
39 old := good
40 old.ID = strings.Repeat("1", 32)
41 old.Verified = false
42 attempt := *good.History
43 attempt.ID = old.ID
44 attempt.Version = "v0"
45 attempt.SHA256 = "old-digest"
46 attempt.Started = attempt.Started.Add(-time.Hour)
47 attempt.PromotedAt = attempt.PromotedAt.Add(-time.Hour)
48 old.History = &attempt
49 if err := writeRollbackState(disk, recoveryResultKey(old), old); err != nil {
50 t.Fatal(err)
51 }
52 b := good.HistoryBinding
53 signingPub, key, err := ed25519.GenerateKey(nil)
54 if err != nil {
55 t.Fatal(err)
56 }
57 projection := &rollbackMemory{files: map[string][]byte{}}
58 now := time.Now().UTC()
59 maintenance := rootHistoryMaintenance{history: h, store: e.store, source: source, checks: c, pub: func() (ed25519.PublicKey, error) { return signingPub, nil }, projection: func() (rootstate.Store, error) { return projection, nil }, now: func() time.Time { return now }}
60 directory, responder, scope := completionReadbackFixture(t, e.store, signingPub, key, b, now)
61 maintenance.readback = responder.publish
62 challenge := func() historyreadback.Request {
63 t.Helper()
64 q, err := historyreadback.RequestStatus(t.Context(), directory, release.RootSigner(key), scope, now)
65 if err != nil {
66 t.Fatal(err)
67 }
68 return q
69 }
70 q := challenge()
71 deps := boundRootHistory(h, source.resolve, e.root.obs)
72 deps.Reconstruct = maintenance.run
73 if err := deps.Reconstruct(t.Context(), b); err == nil {
74 t.Fatal("ambiguous original did not restrict reconstruction result")
75 }
76 if _, err := c.mintRefusals(h, b); err == nil {
77 t.Fatal("unsigned ambiguity allowed mint")
78 }
79 var view release.HistoryMigrationStatus
80 raw, err := projection.Read(release.HistoryStatusKey(b.Account, b.App))
81 if err != nil {
82 t.Fatal(err)
83 }
84 if err := json.Unmarshal(raw, &view); err != nil {
85 t.Fatal(err)
86 }
87 if len(view.Hosts) != 1 || len(view.Hosts[0].Recoveries) != 2 {
88 t.Fatalf("independent evidence missing from status: %+v", view)
89 }
90 grant := release.HistoryReconstruction{Schema: 1, Kind: "carlos-release-history", Identity: view.Identity, Revision: view.Revision, Operation: "reconstruct-with-recovery", IssuedAt: now, Account: b.Account, App: b.App, AppRevision: view.AppRevision, BakeRevision: view.BakeRevision, Hosts: []release.HistoryReconstructionHost{view.Hosts[0].HistoryReconstructionHost}}
91 if replacement {
92 prior := grant
93 prior.Operation = "interrupted-history"
94 prior.Hosts = append([]release.HistoryReconstructionHost(nil), grant.Hosts...)
95 prior.Hosts[0].Source = "previous-source"
96 prior.Holds = map[string]release.HistoryReconstructionHold{"stable": {Window: time.Hour, SetAt: now}}
97 wire, err := release.SignRootEnvelope(release.RootSigner(key), prior)
98 if err != nil {
99 t.Fatal(err)
100 }
101 base := h.Store
102 h.Store = &historyIntentFailure{Store: base}
103 if err := h.Reconstruct(signingPub, wire, b.Account, b.App, now, func(release.HistoryBinding) (string, error) { return "previous-source", nil }); err == nil {
104 t.Fatal("missing interrupted intent")
105 }
106 h.Store = base
107 if err := deps.Reconstruct(t.Context(), b); err == nil {
108 t.Fatal("changed source resumed old intent")
109 }
110 raw, err := projection.Read(release.HistoryStatusKey(b.Account, b.App))
111 if err != nil {
112 t.Fatal(err)
113 }
114 var pending release.HistoryMigrationStatus
115 if err := json.Unmarshal(raw, &pending); err != nil {
116 t.Fatal(err)
117 }
118 if !pending.Pending || pending.PendingDigest == "" {
119 t.Fatal("no pending status")
120 }
121 grant.Replaces = pending.PendingDigest
122 }
123 if err := publishHistoryFixture(t.Context(), e.store, release.RootSigner(key), release.HistoryReconstructionKey(grant.Account, grant.App, grant.Identity, grant.Revision), grant); err != nil {
124 t.Fatal(err)
125 }
126 if err := deps.Reconstruct(t.Context(), b); err == nil {
127 t.Fatal("signed reconstruction erased ambiguity")
128 }
129 receipt, err := historyreadback.ReadReconstruction(t.Context(), directory, q, grant, now)
130 if err != nil {
131 t.Fatal("completed reconstruction lacks exact readback", err)
132 }
133 if len(receipt.Status.RecoverySources.Unresolved) != 1 {
134 t.Fatal("reconstruction cleared original ambiguity")
135 }
136 other := grant
137 other.Operation = "not-accepted"
138 if _, err := historyreadback.ReadReconstruction(t.Context(), directory, q, other, now); err == nil {
139 t.Fatal("different authorization reported complete")
140 }
141 retirement := release.HistoryRecoveryRetirement{Schema: 1, Kind: "carlos-release-recovery-retirement", Identity: view.Identity, Operation: "retire-exact-ambiguity", IssuedAt: now, Binding: b, Source: view.Hosts[0].Source, AttemptID: old.ID, OutcomeDigest: view.Hosts[0].RecoveryDigests[old.ID], VerifiedID: good.ID, VerifiedDigest: view.Hosts[0].RecoveryDigests[good.ID]}
142 if err := publishHistoryFixture(t.Context(), e.store, release.RootSigner(key), release.HistoryRecoveryRetirementKey(retirement.Binding, retirement.Identity, retirement.OutcomeDigest), retirement); err != nil {
143 t.Fatal(err)
144 }
145 q = challenge()
146 if _, err := historyreadback.ReadRetirement(t.Context(), directory, q, retirement, now); err == nil {
147 t.Fatal("delivery counted as completion")
148 }
149 before := len(u.effects)
150 if err := deps.Reconstruct(t.Context(), b); err != nil {
151 t.Fatal(err)
152 }
153 if _, err := historyreadback.ReadRetirement(t.Context(), directory, q, retirement, now); err != nil {
154 t.Fatal("retirement lacks exact readback", err)
155 }
156 if refs, err := c.mintRefusals(h, b); err != nil || len(refs) != 0 {
157 t.Fatalf("exact authorized retirement blocked mint: %+v %v", refs, err)
158 }
159 if len(u.effects) != before {
160 t.Fatal("bookkeeping repeated a process effect")
161 }
162 raw, err = projection.Read(release.HistoryStatusKey(b.Account, b.App))
163 if err != nil {
164 t.Fatal(err)
165 }
166 view = release.HistoryMigrationStatus{}
167 if err := json.Unmarshal(raw, &view); err != nil {
168 t.Fatal(err)
169 }
170 if len(view.Hosts[0].RetiredRecoveries) != 1 || view.Hosts[0].RetiredRecoveries[0] != old.ID {
171 t.Fatalf("retirement not projected: %+v", view)
172 }
173 // A stale settled copy cannot hide an outstanding source journal.
174 old.Settled = false
175 if err := writeRollbackState(disk, recoveryResultKey(old), old); err != nil {
176 t.Fatal(err)
177 }
178 if _, err := c.mintRefusals(h, b); err == nil {
179 t.Fatal("retirement consumed a pending original")
180 }
181 q = challenge()
182 if err := deps.Reconstruct(t.Context(), b); err == nil {
183 t.Fatal("maintenance consumed a pending original")
184 }
185 if _, err := historyreadback.ReadRetirement(t.Context(), directory, q, retirement, now); err == nil {
186 t.Fatal("readback consumed pending original")
187 }
188 raw, err = projection.Read(release.HistoryStatusKey(b.Account, b.App))
189 if err != nil {
190 t.Fatal(err)
191 }
192 view = release.HistoryMigrationStatus{}
193 if err := json.Unmarshal(raw, &view); err != nil {
194 t.Fatal(err)
195 }
196 if view.RecoverySources == nil || !view.RecoverySources.Known || len(view.RecoverySources.Unresolved) != 1 || view.RecoverySources.Unresolved[0].Attempt.ID != old.ID || len(view.Hosts[0].RetiredRecoveries) != 0 {
197 t.Fatalf("pending original hidden by projection: %+v", view)
198 }
199
200 })
201 }
202}
203
204func TestPlan14IndependentRecoveryPublishesStatus(t *testing.T) {
205 for _, state := range []string{"success", "failed rollback", "history unavailable"} {
206 t.Run(state, func(t *testing.T) {
207 e, u, c, source, pub := signedRecoveryFixture(t)
208 rt, err := e.root.reg.Get(observedHost)
209 if err != nil {
210 t.Fatal(err)
211 }
212 h := seedHostRootHistory(t, *rt)
213 projection := &rollbackMemory{files: map[string][]byte{}}
214 m := rootHistoryMaintenance{history: h, store: e.store, source: source, checks: c, pub: func() (ed25519.PublicKey, error) { return pub, nil }, projection: func() (rootstate.Store, error) { return projection, nil }, now: time.Now}
215 deps := boundRootHistory(h, source.resolve, e.root.obs)
216 deps.Reconstruct = m.run
217 c.history = func() (*release.RootHistoryPass, error) {
218 if state == "history unavailable" {
219 return nil, os.ErrPermission
220 }
221 return deps, nil
222 }
223 u.crashNew, u.badRollback = state == "failed rollback", state == "failed rollback"
224 n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink}
225 handled, err := n.recover(t.Context(), rt.Host, c, func(string) string { return "5s" }, e.root.reg)
226 if !handled || (err != nil) != (state == "failed rollback") {
227 t.Fatalf("history changed independent effect: %v %v", handled, err)
228 }
229 if state == "history unavailable" {
230 if u.exe != release.VersionPath(u.opt, rt.Account, "v2", rt.App) {
231 t.Fatal("unavailable history blocked fix")
232 }
233 return
234 }
235 raw, err := projection.Read(release.HistoryStatusKey(rt.Account, rt.App))
236 if err != nil {
237 t.Fatal("independent status absent", err)
238 }
239 var view release.HistoryMigrationStatus
240 if err := json.Unmarshal(raw, &view); err != nil {
241 t.Fatal(err)
242 }
243 if len(view.Hosts) != 1 || len(view.Hosts[0].Recoveries) != 1 {
244 t.Fatalf("wrong projected outcomes: %+v", view)
245 }
246 outcome := view.Hosts[0].Recoveries[0]
247 if !outcome.Settled || outcome.Verified != (state == "success") || outcome.Failed != (state == "failed rollback") {
248 t.Fatalf("wrong outcome: %+v", outcome)
249 }
250 })
251 }
252}
253
254type historyIntentFailure struct{ rootstate.Store }
255
256func (s *historyIntentFailure) Write(name string, raw []byte) error {
257 if err := s.Store.Write(name, raw); err != nil {
258 return err
259 }
260 if name == "reconstruction.state" && strings.Contains(string(raw), `"pending":`) {
261 return os.ErrPermission
262 }
263 return nil
264}
265
266func completionReadbackFixture(t *testing.T, store release.Store, pub ed25519.PublicKey, key ed25519.PrivateKey, b release.HistoryBinding, now time.Time) (hostactors.Directory, historyReadbackPublisher, historyreadback.Scope) {
267 t.Helper()
268 _, evidenceKey, err := ed25519.GenerateKey(nil)
269 if err != nil {
270 t.Fatal(err)
271 }
272 enrollment := hostfloor.Enrollment{Box: "box", Enrollment: strings.Repeat("e", 32), InstanceID: "i-box"}
273 directory := hostactors.Directory{Store: store, PublicKey: pub, Bucket: "bucket"}
274 roster := hostactors.Roster{V: 1, DeploymentBucket: "bucket", Revision: 1, Boxes: map[string]hostactors.Box{"box": {Enrollment: enrollment.Enrollment, Namespace: "platform", BoxID: "box", Label: "box", Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-box"}, EvidenceSHA256: strings.Repeat("b", 64), State: "active", CensusPublicKey: evidenceKey.Public().(ed25519.PublicKey)}}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
275 wire, err := release.SignRootEnvelope(release.RootSigner(key), roster)
276 if err != nil {
277 t.Fatal(err)
278 }
279 if _, err := store.Put(t.Context(), hostactors.Key, wire, ""); err != nil {
280 t.Fatal(err)
281 }
282 responder := historyReadbackPublisher{directory: directory, enrollment: enrollment, key: func() (ed25519.PrivateKey, error) { return evidenceKey, nil }, now: func() time.Time { return now }}
283 return directory, responder, historyreadback.Scope{Bucket: "bucket", Box: enrollment.Box, Enrollment: enrollment.Enrollment, Binding: b}
284}
285