amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostresolve
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "reflect"
8 "strings"
9 "testing"
10
11 "github.com/carlosframework/platform/internal/hostclaim"
12 "github.com/carlosframework/platform/internal/hostfloor"
13 "github.com/carlosframework/platform/internal/hostquarantine"
14 "github.com/carlosframework/platform/internal/release"
15 "github.com/carlosframework/platform/internal/status"
16)
17
18func TestPreparedLossClaimProgressAllowsRetiredReadmission(t *testing.T) {
19 for _, retired := range []bool{false, true} {
20 t.Run(map[bool]string{false: "missing-retirement", true: "retired"}[retired], func(t *testing.T) {
21 ctx := t.Context()
22 s, store, now, host := resolveFixture(t)
23 in := submitted(t, s, now, host)
24 store.fail = true
25 if _, err := s.Declare(ctx, "operator", host, in); err == nil {
26 t.Fatal("claim failure not reached")
27 }
28 store.fail = false
29 prepared, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
30 if err != nil || prepared.Record.State != "prepared" {
31 t.Fatal(prepared, err)
32 }
33 before, err := hostclaim.Read(ctx, store, host)
34 if err != nil {
35 t.Fatal(err)
36 }
37 // The retirement owner completes the exact local chore while quarantined.
38 chore := before.Claim.Obligations[0]
39 progressed, err := hostclaim.Update(ctx, store, before, hostclaim.CompleteObligation(chore.Op, chore.ID))
40 if err != nil {
41 t.Fatal(err)
42 }
43 if _, err = s.Declare(ctx, "operator", host, in); !errors.Is(err, hostclaim.ErrConflict) {
44 t.Fatal("stale declaration must refuse", err)
45 }
46 conflicted, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
47 if err != nil || conflicted.Record.State != "conflicted" {
48 t.Fatal("stale loss not settled", conflicted, err)
49 }
50 want := prepared.Record
51 want.State = "conflicted"
52 if !reflect.DeepEqual(want, conflicted.Record) {
53 t.Fatal("accepted history changed")
54 }
55 if _, err = s.Declare(ctx, "operator", host, in); !errors.Is(err, hostclaim.ErrConflict) {
56 t.Fatal("conflicted retry changed outcome", err)
57 }
58 unchanged, err := hostclaim.Read(ctx, store, host)
59 if err != nil || unchanged.ETag != progressed.ETag {
60 t.Fatal("stale consequences applied", err)
61 }
62 prepareReadmission(t, s, *now, conflicted.Record)
63 if !retired {
64 key, _ := hostquarantine.RetirementKey("dead", conflicted.Record.Enrollment, conflicted.Record.QuarantineRevision)
65 raw, etag, err := store.Get(ctx, key)
66 if err != nil {
67 t.Fatal(err)
68 }
69 var receipt hostquarantine.RetirementCompletion
70 if err = json.Unmarshal(raw, &receipt); err != nil {
71 t.Fatal(err)
72 }
73 receipt.Entries = []hostquarantine.RetirementEntry{}
74 if err := receipt.Validate(); err != nil {
75 t.Fatal(err)
76 }
77 raw, _ = json.Marshal(receipt)
78 if _, err = store.Put(ctx, key, raw, etag); err != nil {
79 t.Fatal(err)
80 }
81 digest := hostfloor.Digest(raw)
82 raw, etag, err = store.Get(ctx, "status/boxes/dead.json")
83 if err != nil {
84 t.Fatal(err)
85 }
86 var report status.Box
87 if err = json.Unmarshal(raw, &report); err != nil {
88 t.Fatal(err)
89 }
90 report.Metrics.RetirementReceiptSHA256 = digest
91 raw, _ = json.Marshal(report)
92 if _, err = store.Put(ctx, "status/boxes/dead.json", raw, etag); err != nil {
93 t.Fatal(err)
94 }
95 }
96 roster, err := s.Directory.Read(ctx)
97 if err != nil {
98 t.Fatal(err)
99 }
100 b := roster.Roster.Boxes["dead"]
101 err = s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs)
102 if (err == nil) != retired {
103 t.Fatal("readmission must require retirement of conflicted loss", err)
104 }
105 after, err := s.Directory.Read(ctx)
106 if err != nil {
107 t.Fatal(err)
108 }
109 state := "quarantined"
110 if retired {
111 state = "active"
112 }
113 if after.Roster.Boxes["dead"].State != state || !reflect.DeepEqual(after.Roster.Boxes["dead"].LossIDs, b.LossIDs) {
114 t.Fatal("state or loss history changed", after)
115 }
116 retained, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
117 if err != nil || !reflect.DeepEqual(retained.Record, conflicted.Record) {
118 t.Fatal("readmission erased history", err)
119 }
120 })
121 }
122}
123
124type fenceReadmitStore struct {
125 release.Store
126 unreadable bool
127 writes int
128}
129
130func (s *fenceReadmitStore) Get(ctx context.Context, key string) ([]byte, string, error) {
131 if s.unreadable && strings.HasPrefix(key, "control/box-fences/") {
132 return nil, "", errors.New("fence unavailable")
133 }
134 return s.Store.Get(ctx, key)
135}
136func (s *fenceReadmitStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) {
137 s.writes++
138 return s.Store.Put(ctx, key, raw, etag)
139}
140func TestReadmitRefusesTerminationAuthorityBeforeMutation(t *testing.T) {
141 for _, fault := range []string{"terminated", "malformed", "wrong-writer", "unreadable"} {
142 t.Run(fault, func(t *testing.T) {
143 ctx := t.Context()
144 s, store, now, host := resolveFixture(t)
145 in := submitted(t, s, now, host)
146 if _, err := s.Declare(ctx, "operator", host, in); err != nil {
147 t.Fatal(err)
148 }
149 loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
150 if err != nil {
151 t.Fatal(err)
152 }
153 prepareReadmission(t, s, *now, loss.Record)
154 roster, err := s.Directory.Read(ctx)
155 if err != nil {
156 t.Fatal(err)
157 }
158 b := roster.Roster.Boxes["dead"]
159 fence := map[string]any{"v": 1, "box": "dead", "enrollment": b.Enrollment, "cloud": b.Cloud, "state": "terminated", "observed_at": *now, "via": "describe", "request_id": "request", "written_by": map[string]string{"box": "dead", "enrollment": b.Enrollment}}
160 if fault == "wrong-writer" {
161 fence["written_by"] = map[string]string{"box": "unknown", "enrollment": b.Enrollment}
162 }
163 raw, _ := json.Marshal(fence)
164 if fault == "malformed" {
165 raw = []byte("{")
166 }
167 if fault != "unreadable" {
168 if err := release.CreateExact(ctx, store, "control/box-fences/"+b.Enrollment+".json", raw); err != nil {
169 t.Fatal(err)
170 }
171 }
172 before, err := hostclaim.Read(ctx, store, host)
173 if err != nil {
174 t.Fatal(err)
175 }
176 wrapped := &fenceReadmitStore{Store: store, unreadable: fault == "unreadable"}
177 s.Directory.Store = wrapped
178 if err := s.Readmit(ctx, "dead", b.Enrollment, b.QuarantineRevision, b.LossIDs); err == nil {
179 t.Fatal("fence authority ignored")
180 }
181 if wrapped.writes != 0 {
182 t.Fatal("mutation before fence refusal", wrapped.writes)
183 }
184 after, err := hostclaim.Read(ctx, store, host)
185 if err != nil || after.ETag != before.ETag {
186 t.Fatal("per-host quarantine changed", err)
187 }
188 unchanged, err := s.Directory.Read(ctx)
189 if err != nil || unchanged.ETag != roster.ETag {
190 t.Fatal("global quarantine changed", err)
191 }
192 if unchanged.Roster.Boxes["dead"].State != "quarantined" || len(after.Claim.LossRefs) != 1 {
193 t.Fatal("quarantine or history removed")
194 }
195 })
196 }
197}
198
199type lossBookkeepingStore struct {
200 release.Store
201 fail bool
202}
203
204func (s *lossBookkeepingStore) Put(ctx context.Context, key string, raw []byte, etag string) (string, error) {
205 if s.fail && etag != "" && strings.HasPrefix(key, "control/host-loss/") {
206 return "", errors.New("loss bookkeeping unavailable")
207 }
208 return s.Store.Put(ctx, key, raw, etag)
209}
210func TestPreparedLossCommitReferenceSurvivesClaimProgress(t *testing.T) {
211 ctx := t.Context()
212 s, store, now, host := resolveFixture(t)
213 in := submitted(t, s, now, host)
214 wrapped := &lossBookkeepingStore{Store: store, fail: true}
215 s.Directory.Store = wrapped
216 if _, err := s.Declare(ctx, "operator", host, in); err == nil {
217 t.Fatal("bookkeeping failure not reached")
218 }
219 loss, err := hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
220 if err != nil || loss.Record.State != "prepared" {
221 t.Fatal(loss, err)
222 }
223 committed, err := hostclaim.Read(ctx, store, host)
224 if err != nil || len(committed.Claim.LossRefs) != 1 || committed.Claim.LossRefs[0] != in.Confirmation {
225 t.Fatal("claim not committed", err)
226 }
227 progressed, err := hostclaim.Update(ctx, store, committed, func(*hostclaim.Claim) error { return nil })
228 if err != nil {
229 t.Fatal(err)
230 }
231 wrapped.fail = false
232 if _, err := s.Declare(ctx, "operator", host, in); err != nil {
233 t.Fatal("successful CAS treated as conflict", err)
234 }
235 loss, err = hostclaim.ReadLossRecord(ctx, store, s.Directory.PublicKey, in.Confirmation)
236 if err != nil || loss.Record.State != "committed" {
237 t.Fatal(loss, err)
238 }
239 after, err := hostclaim.Read(ctx, store, host)
240 if err != nil || after.ETag != progressed.ETag {
241 t.Fatal("commit recovery changed claim", err)
242 }
243}
244