amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostresolve
2
3import (
4 "context"
5 "crypto/rand"
6 "encoding/hex"
7 "encoding/json"
8 "errors"
9 "github.com/carlosframework/platform/internal/hostactors"
10 "github.com/carlosframework/platform/internal/hostclaim"
11 "github.com/carlosframework/platform/internal/hostfence"
12 "github.com/carlosframework/platform/internal/release"
13 "math"
14 "os"
15 "reflect"
16 "slices"
17 "time"
18)
19
20var ErrResolve = errors.New("hostresolve: evidence or operation does not authorize resolution")
21
22type Service struct {
23 Directory hostactors.Directory
24 Signer release.Signer
25 Now func() time.Time
26 Owners map[string]OperationOwner
27}
28type Inspection struct {
29 Challenge json.RawMessage `json:"challenge"`
30 Details hostfence.Challenge `json:"details"`
31}
32
33func nonce() (string, error) {
34 var b [16]byte
35 _, err := rand.Read(b[:])
36 return hex.EncodeToString(b[:]), err
37}
38func (s Service) record(ctx context.Context, c *hostclaim.Claim) (hostfence.RecordObservation, error) {
39 _, etag, err := s.Directory.Store.Get(ctx, c.Holder.Key)
40 if errors.Is(err, os.ErrNotExist) {
41 return hostfence.RecordObservation{Key: c.Holder.Key, CommitState: "absent"}, nil
42 }
43 if err != nil {
44 return hostfence.RecordObservation{}, err
45 }
46 return hostfence.RecordObservation{Key: c.Holder.Key, ETag: etag, CommitState: "present"}, nil
47}
48
49// effects implements only the owners present on the Plan 8 baseline. Moves'
50// target-loss policy must come from the moves owner, never from an HTTP flag.
51func effects(c *hostclaim.Claim, box string) (hostfence.Consequences, hostclaim.LossConsequences, error) {
52 var shown hostfence.Consequences
53 var actual hostclaim.LossConsequences
54 if c == nil || c.Holder == nil {
55 return shown, actual, ErrResolve
56 }
57 actual.RecoveryIntent = json.RawMessage(`{"kind":"continue-parking"}`)
58 switch c.Op.Kind {
59 case hostclaim.KindServing:
60 shown.Transition = "destroyed-local-only"
61 actual.RecoveryIntent = json.RawMessage(`{"kind":"destroyed-local-only"}`)
62 for _, o := range c.Obligations {
63 if o.Kind == "park-local" && o.Box == box {
64 ref := hostclaim.ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement}
65 shown.DischargeLocal = append(shown.DischargeLocal, ref)
66 actual.DischargeLocal = append(actual.DischargeLocal, ref)
67 key := o.Op + "/" + o.ID + "/" + o.Retirement
68 shown.DestroyedStorageAssertion = append(shown.DestroyedStorageAssertion, key)
69 actual.DestroyedStorage = append(actual.DestroyedStorage, key)
70 }
71 }
72 if len(actual.DischargeLocal) == 0 {
73 return shown, actual, ErrResolve
74 }
75 case hostclaim.KindDeleting, hostclaim.KindParking:
76 if c.Op.ID == "" || (!slices.Contains(c.Op.Participants, box) && c.Op.Exec != box) {
77 return shown, actual, ErrResolve
78 }
79 shown.Transition = "continue-parking"
80 shown.PossibleWriteLoss = true
81 shown.PossibleTimerLoss = true
82 actual.RevokeExecution = c.Op.Exec == box
83 actual.RemoveParticipation = slices.Contains(c.Op.Participants, box)
84 actual.WritesPotentiallyLost = true
85 actual.TimersPotentiallyLost = true
86 for _, o := range c.Obligations {
87 if (o.Kind == "park-replica" || o.Kind == "park-generation") && o.Owner == "box:"+box {
88 actual.ReassignRegional = append(actual.ReassignRegional, hostclaim.ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement})
89 shown.Chores = append(shown.Chores, o.Op+"/"+o.ID+"/"+o.Retirement)
90 }
91 }
92 default:
93 return shown, actual, ErrResolve
94 }
95 return shown, actual, nil
96}
97func (s Service) Inspect(ctx context.Context, actor, host, name string) (Inspection, error) {
98 ref, b, err := s.Directory.Resolve(ctx, name)
99 if err != nil {
100 return Inspection{}, err
101 }
102 if b.State == "retired" {
103 return Inspection{}, ErrResolve
104 }
105 o, err := hostclaim.Read(ctx, s.Directory.Store, host)
106 if err != nil {
107 return Inspection{}, err
108 }
109 if o.Absent || o.Claim.Holder == nil {
110 return Inspection{}, ErrResolve
111 }
112 policy, err := s.operationOwner(o.Claim.Op.Kind)
113 if err != nil {
114 return Inspection{}, err
115 }
116 record, err := s.record(ctx, o.Claim)
117 if err != nil {
118 return Inspection{}, err
119 }
120 shown, _, err := inspectOwner(ctx, policy, o, record, ref)
121 if err != nil {
122 return Inspection{}, err
123 }
124 id, err := nonce()
125 if err != nil {
126 return Inspection{}, err
127 }
128 c := o.Claim
129 challenge, err := hostfence.PrepareChallenge(s.Now(), hostfence.Challenge{ID: id, Actor: actor, Host: host, HolderIncarnation: c.Holder.Incarnation, HolderStamp: c.Holder.Stamp, HolderSession: c.Holder.Session, ClaimGen: c.Gen, ClaimETag: o.ETag, Op: hostfence.Operation{ID: c.Op.ID, Kind: c.Op.Kind, Phase: c.Op.Phase}, Record: record, Box: ref, Enrollment: b.Enrollment, Cloud: hostfence.CloudBinding{Provider: b.Cloud.Provider, Account: b.Cloud.Account, Region: b.Cloud.Region, InstanceID: b.Cloud.InstanceID}, Consequences: shown})
130 if err != nil {
131 return Inspection{}, err
132 }
133 wire, err := hostfence.SignChallenge(s.Signer, challenge)
134 return Inspection{wire, challenge}, err
135}
136
137type owner struct{}
138
139func (owner) ValidateDeclareLost(c hostclaim.Claim, d hostclaim.LostDeclaration) error {
140 _, want, err := effects(&c, d.Box)
141 if err != nil {
142 return err
143 }
144 if !reflect.DeepEqual(want, d.Consequences) {
145 return ErrResolve
146 }
147 return nil
148}
149func (s Service) Declare(ctx context.Context, actor, host string, in hostfence.Submission) (hostclaim.Observed, error) {
150 now := s.Now().UTC()
151 var c hostfence.Challenge
152 err := release.VerifyRootEnvelope(s.Directory.PublicKey, in.Challenge, &c)
153 if err != nil {
154 return hostclaim.Observed{}, err
155 }
156 if c.Actor != actor || c.Host != host || in.Confirmation != c.ID || !in.Attestation.MaintainComputeFence || !in.Attestation.ColdQuarantineStartOnly || !reflect.DeepEqual(in.Attestation.Disks, c.Consequences.DestroyedStorageAssertion) {
157 return hostclaim.Observed{}, ErrResolve
158 }
159 // A retry uses the durable accepted declaration even after its claim CAS.
160 // Inspect a new challenge only when no accepted declaration exists.
161 prior, readErr := hostclaim.ReadLossRecord(ctx, s.Directory.Store, s.Directory.PublicKey, c.ID)
162 if readErr == nil {
163 if prior.Record.Actor != actor || prior.Record.Host != host || prior.Record.Enrollment != c.Enrollment {
164 return hostclaim.Observed{}, ErrResolve
165 }
166 return s.finish(ctx, prior)
167 }
168 if !errors.Is(readErr, hostclaim.ErrLossRecordAbsent) {
169 return hostclaim.Observed{}, readErr
170 }
171 // Acceptance survives a crash before either quarantine or loss materialization.
172 acceptance, readErr := s.readAcceptance(ctx, c.ID)
173 if readErr == nil {
174 if acceptance.Loss.Actor != actor || acceptance.Loss.Host != host || acceptance.Loss.Enrollment != c.Enrollment || acceptance.Loss.Box != c.Box || acceptance.Loss.ExpectedClaimETag != c.ClaimETag {
175 return hostclaim.Observed{}, ErrResolve
176 }
177 return s.accept(ctx, acceptance)
178 }
179 if !errors.Is(readErr, os.ErrNotExist) {
180 return hostclaim.Observed{}, readErr
181 }
182 if _, err = hostfence.VerifyChallenge(s.Directory.PublicKey, in.Challenge, now); err != nil {
183 return hostclaim.Observed{}, err
184 }
185 if err = hostfence.ValidateObservation(c, in.Observation, now); err != nil {
186 return hostclaim.Observed{}, err
187 }
188 roster, err := s.Directory.Read(ctx)
189 if err != nil {
190 return hostclaim.Observed{}, err
191 }
192 b, ok := roster.Roster.Boxes[c.Box]
193 if !ok || b.Enrollment != c.Enrollment || b.State == "retired" || b.Cloud != (hostactors.Cloud{Provider: c.Cloud.Provider, Account: c.Cloud.Account, Region: c.Cloud.Region, InstanceID: c.Cloud.InstanceID}) {
194 return hostclaim.Observed{}, ErrResolve
195 }
196 observed, err := hostclaim.Read(ctx, s.Directory.Store, host)
197 if err != nil {
198 return hostclaim.Observed{}, err
199 }
200 if observed.Absent || observed.ETag != c.ClaimETag || observed.Claim.Gen != c.ClaimGen {
201 return hostclaim.Observed{}, ErrResolve
202 }
203 record, err := s.record(ctx, observed.Claim)
204 if err != nil || record != c.Record {
205 return hostclaim.Observed{}, ErrResolve
206 }
207 policy, err := s.operationOwner(observed.Claim.Op.Kind)
208 if err != nil {
209 return hostclaim.Observed{}, err
210 }
211 shown, actual, err := inspectOwner(ctx, policy, observed, record, c.Box)
212 if err != nil || !reflect.DeepEqual(shown, c.Consequences) {
213 return hostclaim.Observed{}, ErrResolve
214 }
215 if b.QuarantineRevision == math.MaxUint64 || slices.Contains(b.LossIDs, c.ID) {
216 return hostclaim.Observed{}, ErrResolve
217 }
218 loss := hostclaim.LossRecord{V: 1, ID: c.ID, State: "prepared", Actor: actor, AcceptedAt: now, Host: host, HolderIncarnation: c.HolderIncarnation, HolderStamp: c.HolderStamp, HolderSession: c.HolderSession, ExpectedClaimGen: c.ClaimGen, ExpectedClaimETag: c.ClaimETag, OpID: c.Op.ID, OpKind: c.Op.Kind, OpPhase: c.Op.Phase, Box: c.Box, Enrollment: c.Enrollment, QuarantineRevision: b.QuarantineRevision + 1, FencedAt: now, ConfirmedConsequences: actual, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}}
219 loss.WriteSources, err = ownerSources(ctx, policy, observed, actual)
220 if err != nil {
221 return hostclaim.Observed{}, err
222 }
223 loss.CloudObservation = hostclaim.LossCloudObservation{ObservedAt: in.Observation.ObservedAt, Account: in.Observation.Account, Region: in.Observation.Region, InstanceID: in.Observation.InstanceID, State: in.Observation.State, RequestID: in.Observation.RequestID}
224 loss.OperatorAttestation = hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true, DestroyedStorage: in.Attestation.Disks}
225 if err := loss.Validate(); err != nil {
226 return hostclaim.Observed{}, err
227 }
228 acceptance = lossAcceptance{V: 1, Bucket: s.Directory.Bucket, RosterETag: roster.ETag, Loss: loss}
229 wire, err := release.SignRootEnvelope(s.Signer, acceptance)
230 if err != nil {
231 return hostclaim.Observed{}, err
232 }
233 key, err := acceptanceKey(c.ID)
234 if err != nil {
235 return hostclaim.Observed{}, err
236 }
237 if err := release.CreateExact(ctx, s.Directory.Store, key, wire); err != nil {
238 return hostclaim.Observed{}, err
239 }
240 return s.accept(ctx, acceptance)
241}
242
243type lossAcceptance struct {
244 V int `json:"v"`
245 Bucket string `json:"bucket"`
246 RosterETag string `json:"roster_etag"`
247 Loss hostclaim.LossRecord `json:"loss"`
248}
249
250func acceptanceKey(id string) (string, error) {
251 if _, err := hostclaim.LossRecordKey(id); err != nil {
252 return "", err
253 }
254 return "control/host-loss-acceptances/" + id + ".json", nil
255}
256func (s Service) readAcceptance(ctx context.Context, id string) (lossAcceptance, error) {
257 key, err := acceptanceKey(id)
258 if err != nil {
259 return lossAcceptance{}, err
260 }
261 raw, _, err := s.Directory.Store.Get(ctx, key)
262 if err != nil {
263 return lossAcceptance{}, err
264 }
265 var a lossAcceptance
266 if release.VerifyRootEnvelope(s.Directory.PublicKey, raw, &a) != nil || a.V != 1 || a.Bucket != s.Directory.Bucket || a.RosterETag == "" || a.Loss.ID != id || a.Loss.State != "prepared" || a.Loss.Validate() != nil {
267 return lossAcceptance{}, ErrResolve
268 }
269 return a, nil
270}
271func (s Service) accept(ctx context.Context, a lossAcceptance) (hostclaim.Observed, error) {
272 if _, err := s.operationOwner(a.Loss.OpKind); err != nil {
273 return hostclaim.Observed{}, err
274 }
275 roster, err := s.Directory.Read(ctx)
276 if err != nil {
277 return hostclaim.Observed{}, err
278 }
279 loss := a.Loss
280 b, ok := roster.Roster.Boxes[loss.Box]
281 if !ok || b.Enrollment != loss.Enrollment || b.State == "retired" || b.Cloud.Account != loss.CloudObservation.Account || b.Cloud.Region != loss.CloudObservation.Region || b.Cloud.InstanceID != loss.CloudObservation.InstanceID {
282 return hostclaim.Observed{}, ErrResolve
283 }
284 if !slices.Contains(b.LossIDs, loss.ID) {
285 // Retry unrelated roster changes with a fresh CAS. A declaration for
286 // this same box defeats the accepted revision and needs new inspection.
287 if b.QuarantineRevision == math.MaxUint64 || b.QuarantineRevision+1 != loss.QuarantineRevision {
288 return hostclaim.Observed{}, ErrResolve
289 }
290 roster, err = s.Directory.Quarantine(ctx, s.Signer, roster, loss.Box, loss.Enrollment, loss.ID)
291 if err != nil {
292 return hostclaim.Observed{}, err
293 }
294 b = roster.Roster.Boxes[loss.Box]
295 }
296 if b.State != "quarantined" || b.QuarantineRevision < loss.QuarantineRevision {
297 return hostclaim.Observed{}, ErrResolve
298 }
299 // The global deny remains in force through every subsequent failure.
300 prepared, err := hostclaim.CreateLossRecord(ctx, s.Directory.Store, s.Signer, loss)
301 if err != nil {
302 return hostclaim.Observed{}, err
303 }
304 return s.finish(ctx, prepared)
305}
306
307func (s Service) finish(ctx context.Context, loss hostclaim.ObservedLossRecord) (hostclaim.Observed, error) {
308 policy, err := s.operationOwner(loss.Record.OpKind)
309 if err != nil {
310 return hostclaim.Observed{}, err
311 }
312 observed, err := hostclaim.Read(ctx, s.Directory.Store, loss.Record.Host)
313 if err != nil {
314 return hostclaim.Observed{}, err
315 }
316 if loss.Record.State == "conflicted" {
317 return hostclaim.Observed{}, hostclaim.ErrConflict
318 }
319 // A retained loss reference proves an earlier successful CAS even if
320 // bookkeeping failed. Otherwise a changed ETag fences every stale retry.
321 if loss.Record.State == "prepared" && (observed.Absent || !slices.Contains(observed.Claim.LossRefs, loss.Record.ID)) && (observed.Absent || observed.ETag != loss.Record.ExpectedClaimETag || observed.Claim.Gen != loss.Record.ExpectedClaimGen) {
322 r := loss.Record
323 r.State = "conflicted"
324 if _, err := hostclaim.ReplaceLossRecord(ctx, s.Directory.Store, s.Signer, loss, r); err != nil {
325 return hostclaim.Observed{}, err
326 }
327 return hostclaim.Observed{}, hostclaim.ErrConflict
328 }
329 if loss.Record.State == "prepared" && !slices.Contains(observed.Claim.LossRefs, loss.Record.ID) {
330 if err := ownerHook(ctx, policy, observed, loss.Record, false); err != nil {
331 return hostclaim.Observed{}, err
332 }
333 }
334 next, err := hostclaim.DeclareLost(ctx, s.Directory.Store, s.Directory.PublicKey, observed, loss.Record.ID, policy)
335 if err != nil {
336 return hostclaim.Observed{}, err
337 }
338 if loss.Record.State == "committed" {
339 return next, ownerHook(ctx, policy, next, loss.Record, true)
340 }
341 r := loss.Record
342 r.State = "committed"
343 r.ResultClaimGen = next.Claim.Gen
344 _, err = hostclaim.ReplaceLossRecord(ctx, s.Directory.Store, s.Signer, loss, r)
345 if err != nil {
346 return next, err
347 }
348 return next, ownerHook(ctx, policy, next, r, true)
349}
350