amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostops
2
3import (
4 "context"
5 "crypto/sha256"
6 "encoding/hex"
7 "encoding/json"
8 "errors"
9 "fmt"
10 "os"
11 "reflect"
12 "strconv"
13 "strings"
14 "time"
15
16 "github.com/carlosframework/platform/internal/hostactors"
17 "github.com/carlosframework/platform/internal/hostbox"
18 "github.com/carlosframework/platform/internal/hostclaim"
19 "github.com/carlosframework/platform/internal/hostevidence"
20 "github.com/carlosframework/platform/internal/hostfence"
21 "github.com/carlosframework/platform/internal/lease"
22 "github.com/carlosframework/platform/internal/release"
23 "github.com/carlosframework/platform/internal/termfence"
24)
25
26type StopReceipt struct {
27 RequestLimitNS int64 `json:"request_limit_ns"`
28 Region string `json:"region,omitempty"`
29 ClaimETag string `json:"claim_etag,omitempty"`
30 ObservedBootID string `json:"observed_boot_id,omitempty"`
31 V int `json:"v"`
32 ID string `json:"id"`
33 Host string `json:"host"`
34 Incarnation string `json:"incarnation"`
35 InstallationStamp uint64 `json:"installation_stamp"`
36 WriterSession uint64 `json:"writer_session"`
37 OperationID string `json:"operation_id"`
38 Phase string `json:"phase"`
39 ObligationID string `json:"obligation_id"`
40 Box string `json:"box"`
41 Enrollment string `json:"enrollment"`
42 BootID string `json:"boot_id"`
43 ControllerID string `json:"controller_id"`
44 ClaimGen uint64 `json:"claim_gen"`
45 DirectoryRevision string `json:"directory_revision"`
46 Kind string `json:"kind"`
47 EvidenceAt time.Time `json:"evidence_at"`
48 FenceID string `json:"fence_id"`
49 InventorySHA256 string `json:"inventory_sha256"`
50 LocalJournalSHA256 string `json:"local_journal_sha256"`
51 ProducerDrainDigest string `json:"producer_drain_digest"`
52 CgroupIdentities []string `json:"cgroup_identities"`
53 UnitInvocationIDs []string `json:"unit_invocation_ids"`
54}
55
56func evidenceDigest(raw []byte) string { sum := sha256.Sum256(raw); return hex.EncodeToString(sum[:]) }
57func digestOK(s string) bool {
58 raw, err := hex.DecodeString(s)
59 return err == nil && len(raw) == 32 && s == strings.ToLower(s)
60}
61
62func (a ProductionAuthority) signedRoster(ctx context.Context) (hostactors.Roster, []byte, error) {
63 raw, _, err := a.Store.Get(ctx, hostactors.Key)
64 if err != nil {
65 return hostactors.Roster{}, nil, err
66 }
67 var roster hostactors.Roster
68 if release.VerifyRootEnvelope(a.Directory.PublicKey, raw, &roster) != nil || roster.Validate(a.Directory.Bucket) != nil || roster.RequestLimitNS != int64(hostbox.RequestLimit) {
69 return roster, nil, ErrContract
70 }
71 return roster, raw, nil
72}
73func (a ProductionAuthority) verifyEvidence(ctx context.Context, obs hostclaim.Observed, obID, opID string, ref hostclaim.RetirementEvidenceRef, bind bool) (time.Time, error) {
74 if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || ref.Box == "" || ref.Enrollment == "" || ref.OperationID != opID || ref.ObligationID != obID || ref.EvidenceAt.IsZero() {
75 return time.Time{}, ErrIdentity
76 }
77 raw, _, err := a.Store.Get(ctx, ref.Key)
78 if err != nil {
79 return time.Time{}, err
80 }
81 if evidenceDigest(raw) != ref.SHA256 {
82 return time.Time{}, ErrIdentity
83 }
84 if ref.Kind == "termination" {
85 fence := hostclaim.FenceRef{Enrollment: ref.Enrollment, Key: ref.Key, SHA256: ref.SHA256}
86 if err = hostclaim.VerifyFenceForBox(ctx, a.Store, hostfence.ClaimFenceAuthority{Directory: a.Directory, Now: a.Now}, fence, ref.Box); err != nil {
87 return time.Time{}, err
88 }
89 var record termfence.Record
90 if json.Unmarshal(raw, &record) != nil {
91 return time.Time{}, ErrIdentity
92 }
93 at, err := time.Parse(time.RFC3339Nano, record.ObservedAt)
94 if err != nil || !at.Equal(ref.EvidenceAt) {
95 return time.Time{}, ErrIdentity
96 }
97 return at, nil
98 }
99 if ref.Kind != "receipt" {
100 return time.Time{}, ErrIdentity
101 }
102 var roster hostactors.Roster
103 if release.VerifyRootEnvelope(a.Directory.PublicKey, ref.SignedRoster, &roster) != nil || roster.Validate(a.Directory.Bucket) != nil || roster.RequestLimitNS != int64(hostbox.RequestLimit) {
104 return time.Time{}, ErrIdentity
105 }
106 actor, ok := roster.Boxes[ref.Box]
107 if !ok || actor.Enrollment != ref.Enrollment {
108 return time.Time{}, ErrIdentity
109 }
110 signature, _, err := a.Store.Get(ctx, ref.Key+".sig")
111 if err != nil {
112 return time.Time{}, err
113 }
114 if evidenceDigest(signature) != ref.SignatureSHA256 || hostevidence.Verify(actor.CensusPublicKey, ref.Key, raw, signature) != nil {
115 return time.Time{}, ErrIdentity
116 }
117 var r StopReceipt
118 if decode(raw, &r) != nil {
119 return time.Time{}, ErrIdentity
120 }
121 h := obs.Claim.Holder
122 prefix := fmt.Sprintf("control/hosts/%s/evidence/%s/%s/", obs.Host, opID, ref.Box)
123 revision, err := strconv.ParseUint(r.DirectoryRevision, 10, 64)
124 if err != nil || revision == 0 || revision > roster.Revision || r.V != 1 || !safeSegment(r.ID) || ref.Key != prefix+r.ID+".json" || r.Host != obs.Host || r.Incarnation != h.Incarnation || r.OperationID != opID || r.Box != ref.Box || r.Enrollment != ref.Enrollment || r.ClaimGen == 0 || r.ClaimGen > obs.Claim.Gen || r.BootID == "" || r.ControllerID == "" || r.FenceID == "" || r.Phase == "" || r.ObligationID == "" || !r.EvidenceAt.Equal(ref.EvidenceAt) || r.EvidenceAt.After(a.Now().Add(5*time.Second)) || r.RequestLimitNS != int64(hostbox.RequestLimit) || !digestOK(r.InventorySHA256) || !digestOK(r.LocalJournalSHA256) || !digestOK(r.ProducerDrainDigest) {
125 return time.Time{}, ErrIdentity
126 }
127 if ref.SourceObligationID == "" || r.ObligationID != ref.SourceObligationID {
128 return time.Time{}, ErrIdentity
129 }
130 if r.Kind != "stopped" && r.Kind != "empty" {
131 return time.Time{}, ErrIdentity
132 }
133 if r.InstallationStamp != ref.InstallationStamp || r.WriterSession != ref.WriterSession {
134 return time.Time{}, ErrIdentity
135 }
136 if bind && (ref.Box == h.WriterBox || ref.Box == obs.Claim.Op.Exec) {
137 if r.Kind != "stopped" || r.InstallationStamp != h.Stamp || r.WriterSession != h.Session || len(r.CgroupIdentities) == 0 {
138 return time.Time{}, ErrIdentity
139 }
140 } else if bind && r.Kind == "stopped" {
141 matched := false
142 if inventory, ok := obs.Claim.Inventory[ref.Box]; ok {
143 for _, stamp := range inventory.Paths {
144 if stamp == r.InstallationStamp {
145 matched = true
146 }
147 }
148 }
149 if !matched {
150 return time.Time{}, ErrPending
151 }
152 }
153 return r.EvidenceAt, nil
154}
155
156func (a ProductionAuthority) chooseEvidence(ctx context.Context, obs hostclaim.Observed, obID, opID, box string) (hostclaim.RetirementEvidenceRef, error) {
157 roster, signed, err := a.signedRoster(ctx)
158 if err != nil {
159 return hostclaim.RetirementEvidenceRef{}, err
160 }
161 actor, ok := roster.Boxes[box]
162 if !ok {
163 return hostclaim.RetirementEvidenceRef{}, ErrIdentity
164 }
165 prefix := fmt.Sprintf("control/hosts/%s/evidence/%s/%s/", obs.Host, opID, box)
166 keys, err := a.Store.List(ctx, prefix)
167 if err != nil {
168 return hostclaim.RetirementEvidenceRef{}, err
169 }
170 var chosen hostclaim.RetirementEvidenceRef
171 for _, key := range keys {
172 if !strings.HasPrefix(key, prefix) || strings.Contains(strings.TrimPrefix(key, prefix), "/") {
173 return chosen, ErrIdentity
174 }
175 if !strings.HasSuffix(key, ".json") || strings.HasSuffix(key, "-retirement.json") {
176 continue
177 }
178 raw, _, err := a.Store.Get(ctx, key)
179 if err != nil {
180 return chosen, err
181 }
182 var receipt StopReceipt
183 if json.Unmarshal(raw, &receipt) != nil {
184 return chosen, ErrIdentity
185 }
186 if receipt.OperationID != opID || receipt.Host != obs.Host || receipt.Box != box || receipt.Incarnation != obs.Claim.Holder.Incarnation || receipt.Enrollment != actor.Enrollment {
187 continue
188 }
189 signature, _, err := a.Store.Get(ctx, key+".sig")
190 if errors.Is(err, os.ErrNotExist) {
191 continue
192 }
193 if err != nil {
194 return chosen, err
195 }
196 ref := hostclaim.RetirementEvidenceRef{SourceObligationID: receipt.ObligationID, InstallationStamp: receipt.InstallationStamp, WriterSession: receipt.WriterSession, Kind: "receipt", Box: box, Enrollment: actor.Enrollment, OperationID: opID, ObligationID: obID, Key: key, SHA256: evidenceDigest(raw), SignatureSHA256: evidenceDigest(signature), SignedRoster: signed, EvidenceAt: receipt.EvidenceAt}
197 if _, err = a.verifyEvidence(ctx, obs, obID, opID, ref, true); err != nil {
198 return chosen, err
199 }
200 // Earliest complete proof is stable once cited. A retry never replaces it
201 // with a newly published receipt carrying a later wall-clock time.
202 if chosen.Key == "" || ref.EvidenceAt.Before(chosen.EvidenceAt) || (ref.EvidenceAt.Equal(chosen.EvidenceAt) && ref.Key < chosen.Key) {
203 chosen = ref
204 }
205 }
206 if chosen.Key != "" {
207 return chosen, nil
208 }
209 if opID != obs.Claim.Op.ID {
210 return chosen, ErrPending
211 }
212 var fence hostclaim.FenceRef
213 if json.Unmarshal([]byte(obs.Claim.Op.Journal["fence."+box]), &fence) != nil {
214 return chosen, ErrPending
215 }
216 if err = hostclaim.VerifyFenceForBox(ctx, a.Store, hostfence.ClaimFenceAuthority{Directory: a.Directory, Now: a.Now}, fence, box); err != nil {
217 return chosen, err
218 }
219 raw, _, err := a.Store.Get(ctx, fence.Key)
220 if err != nil {
221 return chosen, err
222 }
223 if evidenceDigest(raw) != fence.SHA256 {
224 return chosen, ErrIdentity
225 }
226 var record termfence.Record
227 if json.Unmarshal(raw, &record) != nil {
228 return chosen, ErrIdentity
229 }
230 at, err := time.Parse(time.RFC3339Nano, record.ObservedAt)
231 if err != nil {
232 return chosen, err
233 }
234 chosen = hostclaim.RetirementEvidenceRef{Kind: "termination", Box: box, Enrollment: fence.Enrollment, OperationID: opID, ObligationID: obID, Key: fence.Key, SHA256: fence.SHA256, EvidenceAt: at}
235 return chosen, nil
236}
237
238// Discovery retains its original writer census even after local chores finish.
239// The serving box remains live and is excluded only for that persisted census.
240func (a ProductionAuthority) evidenceBoxes(ctx context.Context, obs hostclaim.Observed, opID string) ([]string, error) {
241 if obs.Claim != nil && obs.Claim.Touched.DiscoveryOp == opID && opID != "" {
242 if !obs.Claim.Touched.DiscoveryBound {
243 return nil, ErrPending
244 }
245 return append([]string(nil), obs.Claim.Touched.DiscoveryBoxes...), nil
246 }
247 _, boxes, err := a.Inventory(ctx, obs)
248 return boxes, err
249}
250
251func (a ProductionAuthority) collectEvidence(ctx context.Context, obs hostclaim.Observed, obID, opID string) (map[string]hostclaim.RetirementEvidenceRef, time.Time, error) {
252 boxes, err := a.evidenceBoxes(ctx, obs, opID)
253 if err != nil {
254 return nil, time.Time{}, err
255 }
256 if len(boxes) == 0 {
257 return nil, time.Time{}, ErrPending
258 }
259 refs := map[string]hostclaim.RetirementEvidenceRef{}
260 var latest time.Time
261 for _, box := range boxes {
262 ref, err := a.chooseEvidence(ctx, obs, obID, opID, box)
263 if err != nil {
264 return nil, time.Time{}, err
265 }
266 refs[box] = ref
267 if ref.EvidenceAt.After(latest) {
268 latest = ref.EvidenceAt
269 }
270 }
271 return refs, latest, nil
272}
273func (a ProductionAuthority) verifyReferences(ctx context.Context, obs hostclaim.Observed, obID, opID string, refs map[string]hostclaim.RetirementEvidenceRef) (time.Time, error) {
274 boxes, err := a.evidenceBoxes(ctx, obs, opID)
275 if err != nil {
276 return time.Time{}, err
277 }
278 if len(boxes) == 0 || len(boxes) != len(refs) {
279 return time.Time{}, ErrPending
280 }
281 var latest time.Time
282 for _, box := range boxes {
283 ref, ok := refs[box]
284 if !ok || ref.Box != box {
285 return time.Time{}, ErrIdentity
286 }
287 at, err := a.verifyEvidence(ctx, obs, obID, opID, ref, false)
288 if err != nil {
289 return time.Time{}, err
290 }
291 if at.After(latest) {
292 latest = at
293 }
294 }
295 return latest, nil
296}
297func (a ProductionAuthority) StopBeforeTeardown(ctx context.Context, obs hostclaim.Observed) error {
298 if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil {
299 return ErrIdentity
300 }
301 if raw := obs.Claim.Op.Journal["stop_evidence"]; raw != "" {
302 var refs map[string]hostclaim.RetirementEvidenceRef
303 if json.Unmarshal([]byte(raw), &refs) != nil {
304 return ErrIdentity
305 }
306 _, err := a.verifyReferences(ctx, obs, "", obs.Claim.Op.ID, refs)
307 return err
308 }
309 refs, _, err := a.collectEvidence(ctx, obs, "", obs.Claim.Op.ID)
310 if err != nil {
311 return err
312 }
313 raw, err := json.Marshal(refs)
314 if err != nil {
315 return err
316 }
317 _, err = hostclaim.Update(ctx, a.Store, obs, func(c *hostclaim.Claim) error {
318 if c.Op.ID != obs.Claim.Op.ID || c.Holder == nil || c.Holder.Incarnation != obs.Claim.Holder.Incarnation {
319 return ErrIdentity
320 }
321 if c.Op.Journal == nil {
322 c.Op.Journal = map[string]string{}
323 }
324 c.Op.Journal["stop_evidence"] = string(raw)
325 return nil
326 })
327 if err != nil {
328 return err
329 }
330 // Force the caller to re-read the claim carrying the durable references.
331 return ErrPending
332}
333
334type RetirementEvidence struct{ Authority ProductionAuthority }
335
336func (e RetirementEvidence) VerifyObligation(ctx context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) (hostclaim.ParkScope, error) {
337 if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || ob.Op == "" || ob.Region == "" || ob.Owner != "region:"+ob.Region || (ob.Kind != "park-replica" && ob.Kind != "park-generation") {
338 return hostclaim.ParkScope{}, ErrIdentity
339 }
340 if ob.Park != nil {
341 if err := e.VerifyBound(ctx, obs, ob); err != nil {
342 return hostclaim.ParkScope{}, err
343 }
344 return *ob.Park, nil
345 }
346 if ob.ParkTarget == "" {
347 return hostclaim.ParkScope{}, ErrPending
348 }
349 refs, _, err := e.Authority.collectEvidence(ctx, obs, ob.ID, ob.Op)
350 if err != nil {
351 return hostclaim.ParkScope{}, err
352 }
353 scope, err := retirementScope(obs.Host, obs.Claim.Holder.Incarnation, ob.ParkTarget, refs)
354 if err != nil {
355 return scope, err
356 }
357 _, err = hostclaim.Update(ctx, e.Authority.Store, obs, func(c *hostclaim.Claim) error {
358 for i := range c.Obligations {
359 current := &c.Obligations[i]
360 if current.ID == ob.ID && current.Op == ob.Op {
361 if !reflect.DeepEqual(*current, ob) {
362 return ErrPending
363 }
364 current.EvidenceRefs = refs
365 current.Park = &scope
366 return nil
367 }
368 }
369 return ErrPending
370 })
371 return scope, err
372}
373func (e RetirementEvidence) VerifyBound(ctx context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) error {
374 if ob.Park == nil || ob.ParkTarget == "" || ob.Park.Location != ob.ParkTarget || ob.Park.RequestLimit != hostbox.RequestLimit || obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || ob.Park.Incarnation != obs.Claim.Holder.Incarnation {
375 return ErrIdentity
376 }
377 latest, err := e.Authority.verifyReferences(ctx, obs, ob.ID, ob.Op, ob.EvidenceRefs)
378 if err != nil {
379 return err
380 }
381 if !latest.Equal(ob.Park.EvidenceAt) {
382 return ErrIdentity
383 }
384 return ob.Park.Validate(obs.Host)
385}
386func (e RetirementEvidence) VerifyRetirement(context.Context, hostclaim.Observed, []lease.ObjectVersion) (string, time.Time, error) {
387 // At-rest/holderless residue needs Plan6b's archived-incarnation census.
388 // Current holder evidence does not authorize another generation's sweep.
389 return "", time.Time{}, ErrPending
390}
391
392type combinedRegionalEvidence struct {
393 RetirementEvidence
394 Previous hostbox.RegionalEvidence
395}
396
397func (e combinedRegionalEvidence) VerifyObligation(ctx context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) (hostclaim.ParkScope, error) {
398 // Marked retirement/discovery work must never borrow the older protocol.
399 if ob.ParkTarget != "" || len(ob.EvidenceRefs) != 0 {
400 return e.RetirementEvidence.VerifyObligation(ctx, obs, ob)
401 }
402 previous, ok := e.Previous.(hostbox.RegionalObligationEvidence)
403 if !ok || previous == nil {
404 return hostclaim.ParkScope{}, ErrContract
405 }
406 return previous.VerifyObligation(ctx, obs, ob)
407}
408func (e combinedRegionalEvidence) VerifyRetirement(ctx context.Context, obs hostclaim.Observed, versions []lease.ObjectVersion) (string, time.Time, error) {
409 if e.Previous == nil {
410 return "", time.Time{}, ErrPending
411 }
412 return e.Previous.VerifyRetirement(ctx, obs, versions)
413}
414
415func NewClaimRegionalWorker(regional hostbox.Regional, enrollment hostbox.RegionalEnrollment, authority ProductionAuthority, prior ...hostbox.RegionalEvidence) (hostbox.RegionalWorker, error) {
416 if len(prior) > 1 || (len(prior) == 1 && prior[0] == nil) {
417 return hostbox.RegionalWorker{}, ErrContract
418 }
419 var original hostbox.RegionalEvidence
420 if len(prior) == 1 {
421 original = prior[0]
422 }
423 if enrollment == nil || regional.Owner == nil || regional.Engine.Authorize == nil || regional.Engine.Leases == nil || regional.Engine.View == nil || regional.Engine.Claims == nil || authority.Store == nil || authority.Now == nil {
424 return hostbox.RegionalWorker{}, ErrContract
425 }
426 switch regional.Engine.Store.(type) {
427 case hostbox.RequestStore, *hostbox.RequestStore:
428 default:
429 return hostbox.RegionalWorker{}, ErrContract
430 }
431 provider := RetirementEvidence{Authority: authority}
432 previous := regional.Engine.Authorize
433 regional.Engine.Authorize = func(ctx context.Context, obs hostclaim.Observed, ob hostclaim.Obligation) error {
434 if err := previous(ctx, obs, ob); err != nil {
435 return err
436 }
437 // A pre-bound move/sweep owns its original evidence protocol. The
438 // caller's mandatory authorization above revalidates that protocol.
439 // Never route a newly produced retirement chore through this branch.
440 if ob.Park != nil && ob.ParkTarget == "" && len(ob.EvidenceRefs) == 0 {
441 if ob.Op == "" || ob.Park.RequestLimit != hostbox.RequestLimit {
442 return ErrIdentity
443 }
444 return ob.Park.Validate(obs.Host)
445 }
446 return provider.VerifyBound(ctx, obs, ob)
447 }
448 return hostbox.RegionalWorker{Regional: regional, Enrollment: enrollment, Evidence: combinedRegionalEvidence{RetirementEvidence: provider, Previous: original}}, nil
449}
450