amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostops
2
3import (
4 "bytes"
5 "context"
6 "crypto/ed25519"
7 "crypto/sha256"
8 "encoding/base32"
9 "encoding/hex"
10 "encoding/json"
11 "errors"
12 "fmt"
13 "github.com/carlosframework/platform/internal/park"
14 "io"
15 "reflect"
16 "strconv"
17 "strings"
18 "time"
19
20 "github.com/carlosframework/platform/internal/hostclaim"
21 "github.com/carlosframework/platform/internal/hostfence"
22 "github.com/carlosframework/platform/internal/hostresolve"
23 "github.com/carlosframework/platform/internal/moves"
24 "github.com/carlosframework/platform/internal/provision"
25 "github.com/carlosframework/platform/internal/release"
26)
27
28// The moves owner of a declared-lost target (moves spec §7.10, claim Plan 10
29// J4). hostresolve owns the signed challenge, the acceptance, the roster
30// quarantine and the loss record; this owner decides which target loss is
31// legal, what it costs, and completes the move's own record and claim
32// consequences around the generic claim CAS:
33//
34// - InspectLoss (read-only): the exact target, the record observation and
35// the effects, with every value recovery needs frozen in the signed
36// recovery intent;
37// - LossWriteSources (pure): every target session since the cut and an
38// uncovered source cut, frozen into the signed record;
39// - ValidateDeclareLost (pure): the signed effects are exactly what this
40// claim implies;
41// - PrepareLoss: the record's rollback decision with the target lost,
42// before the claim CAS, so no commit can follow the declaration;
43// - FinishLoss: the claim's release of the lost target, then the record's
44// clean release with the loss as its evidence.
45
46const (
47 moveLossIntentKind = "move-target-lost"
48 // moveLossTransition is the inspection's machine-readable transition.
49 moveLossTransition = "move-target-lost"
50 // moveLossRequestLimit bounds a park's request window for the lost
51 // target's scopes, as the box's own release bounds its (cutRequestLimit).
52 moveLossRequestLimit = 2 * time.Minute
53
54 moveLossRecordLanding = "landing"
55 moveLossRecordRollingBack = "rolling_back"
56)
57
58// Move kinds share the exact local-only cleanup owner. Data-authority
59// target loss remains restricted to landing before commit and restoring
60// before source takeover; moving and reverting gain no takeover authority.
61var moveLossKinds = []string{hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindReverting, hostclaim.KindRestoring}
62
63// moveLossIntent is the signed recovery intent: everything the hooks and a
64// crash recovery need, bound to the inspected claim and record.
65type moveLossIntent struct {
66 Kind string `json:"kind"`
67 Move string `json:"move"`
68 Incarnation string `json:"incarnation"`
69 Rollback string `json:"rollback"`
70 Target string `json:"target"`
71 Source string `json:"source"`
72 TargetRegion string `json:"target_region"`
73 SourceRegion string `json:"source_region"`
74 RecordKey string `json:"record_key"`
75 RecordETag string `json:"record_etag"`
76 RecordSeq uint64 `json:"record_seq"`
77 RecordState string `json:"record_state"`
78 TargetPhase string `json:"target_phase,omitempty"`
79 Dirty bool `json:"dirty"`
80 ReturnLane bool `json:"return_lane"`
81 SrcCovered bool `json:"src_covered"`
82 CutSession uint64 `json:"cut_session"`
83 SrcGen string `json:"src_gen,omitempty"`
84 // Supersedes is the loss record key a lost record still names when this
85 // declaration rebinds it: that declaration's claim CAS never landed and
86 // its signed record is conflicted (B1). TargetPhase, Dirty and
87 // ReturnLane are then that declaration's signed values, since lost has
88 // erased the record's own.
89 Supersedes string `json:"supersedes,omitempty"`
90}
91
92func (in moveLossIntent) encode() (json.RawMessage, error) {
93 raw, err := json.Marshal(in)
94 if err != nil {
95 return nil, err
96 }
97 return canonicalJSON(raw)
98}
99
100// canonicalJSON re-encodes one JSON value with sorted keys and exact
101// numbers, so the signed intent and every recomputation compare as bytes.
102func canonicalJSON(raw []byte) (json.RawMessage, error) {
103 dec := json.NewDecoder(bytes.NewReader(raw))
104 dec.UseNumber()
105 var v any
106 if err := dec.Decode(&v); err != nil {
107 return nil, err
108 }
109 if _, err := dec.Token(); err != io.EOF {
110 return nil, ErrIdentity
111 }
112 return json.Marshal(v)
113}
114
115func decodeMoveLossIntent(raw json.RawMessage) (moveLossIntent, error) {
116 dec := json.NewDecoder(bytes.NewReader(raw))
117 dec.DisallowUnknownFields()
118 var in moveLossIntent
119 if err := dec.Decode(&in); err != nil {
120 return moveLossIntent{}, hostresolve.ErrResolve
121 }
122 if _, err := dec.Token(); err != io.EOF {
123 return moveLossIntent{}, hostresolve.ErrResolve
124 }
125 if in.Kind != moveLossIntentKind || in.Move == "" || in.Incarnation == "" || !provision.ValidRollbackID(in.Rollback) || in.Target == "" || in.Source == "" ||
126 in.TargetRegion == "" || in.SourceRegion == "" || in.RecordKey == "" || in.RecordETag == "" ||
127 (in.RecordState != moveLossRecordLanding && in.RecordState != moveLossRecordRollingBack) {
128 return moveLossIntent{}, hostresolve.ErrResolve
129 }
130 return in, nil
131}
132
133// moveLossRollbackID is the rollback a landing's declaration starts. It is
134// derived from the inspected claim observation, so a recomputation inside
135// the claim CAS (ValidateDeclareLost) and every replay agree on it.
136func moveLossRollbackID(opID, claimETag, box string) string {
137 sum := sha256.Sum256([]byte(moveLossIntentKind + "\x00" + opID + "\x00" + claimETag + "\x00" + box))
138 enc := strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(sum[:]))
139 return "rb_" + enc[:26]
140}
141
142// moveLossNewID mints the declaration's chore ids deterministically, so the
143// scoped store can recompute the release's claim write byte for byte.
144func moveLossNewID(lossID string) func() string {
145 n := 0
146 return func() string {
147 n++
148 sum := sha256.Sum256([]byte(moveLossIntentKind + "\x00" + lossID + "\x00" + strconv.Itoa(n)))
149 return hex.EncodeToString(sum[:16])
150 }
151}
152
153// moveLossRefusal is a refusal the move door answers with its recorded
154// sentence (phase1Conflict); anything else is the door's fixed fallback.
155func moveLossRefusal(sentence string) error {
156 return fmt.Errorf("%w: %s", provision.ErrMoveConflict, sentence)
157}
158
159// claimLossIntent is the claim's half of a legal target loss: the box is
160// the move's exact target, a participant and not the source; the op is a
161// landing before its commit or this move's rollback before the source took
162// over; the seal says whether the source's own replica covered its cut.
163func claimLossIntent(c *hostclaim.Claim, box string) (moveLossIntent, error) {
164 if c == nil || c.Holder == nil || c.Holder.Family != "instance" || box == "" {
165 return moveLossIntent{}, hostresolve.ErrResolve
166 }
167 op := c.Op
168 source, target := op.Intent["source"], op.Intent["target"]
169 if op.ID == "" || op.Intent["admission_id"] != op.ID || source == "" || target == "" || source == target || op.Intent["to"] == "" {
170 return moveLossIntent{}, hostresolve.ErrResolve
171 }
172 if box == source || box != target {
173 return moveLossIntent{}, moveLossRefusal("the source box cannot be declared lost by a move")
174 }
175 if len(op.Participants) != 2 || op.Participants[0] != source || op.Participants[1] != target {
176 return moveLossIntent{}, hostresolve.ErrResolve
177 }
178 for _, lost := range c.Lost {
179 if lost == box {
180 return moveLossIntent{}, hostresolve.ErrResolve
181 }
182 }
183 in := moveLossIntent{Kind: moveLossIntentKind, Move: op.ID, Incarnation: c.Holder.Incarnation, Target: target, Source: source, TargetRegion: op.Intent["to"], RecordKey: c.Holder.Key}
184 switch {
185 case op.Kind == hostclaim.KindLanding && op.Phase == hostclaim.PhaseCommitted:
186 return moveLossIntent{}, moveLossRefusal(provision.RefusedCommitted)
187 case op.Kind == hostclaim.KindLanding:
188 if op.Exec != target {
189 return moveLossIntent{}, hostresolve.ErrResolve
190 }
191 case op.Kind == hostclaim.KindRestoring && op.Phase == hostclaim.PhaseReleasing:
192 if op.Exec != target || !provision.ValidRollbackID(op.Intent["rollback"]) {
193 return moveLossIntent{}, hostresolve.ErrResolve
194 }
195 in.Rollback = op.Intent["rollback"]
196 case op.Kind == hostclaim.KindRestoring && op.Phase == hostclaim.PhaseReleased:
197 if op.Exec != "" || op.Journal[hostclaim.JournalEvidenceTarget] == "" || !provision.ValidRollbackID(op.Intent["rollback"]) {
198 return moveLossIntent{}, hostresolve.ErrResolve
199 }
200 in.Rollback = op.Intent["rollback"]
201 case op.Kind == hostclaim.KindRestoring:
202 return moveLossIntent{}, moveLossRefusal(provision.RefusedNothingToDeclare)
203 default:
204 return moveLossIntent{}, hostresolve.ErrResolve
205 }
206 seal, err := moves.DecodeSeal(op.Journal[hostclaim.JournalSeal])
207 if err != nil {
208 return moveLossIntent{}, hostresolve.ErrResolve
209 }
210 // The cut is a paired identity: a real session with its generation, or
211 // the legacy {session 0, generation ""} of a source that never had one
212 // (claim spec §6.4a). A missing key or a mismatched pair is refused.
213 rawCut, ok := op.Journal[hostclaim.JournalCutSession]
214 if !ok {
215 return moveLossIntent{}, hostresolve.ErrResolve
216 }
217 cut, err := strconv.ParseUint(rawCut, 10, 64)
218 if err != nil {
219 return moveLossIntent{}, hostresolve.ErrResolve
220 }
221 in.SrcCovered, in.CutSession = seal.SrcCovered, cut
222 if !seal.SrcCovered {
223 gen, ok := op.Journal[hostclaim.JournalSrcGen]
224 if !ok || (cut == 0) != (gen == "") {
225 return moveLossIntent{}, hostresolve.ErrResolve
226 }
227 in.SrcGen = gen
228 }
229 return in, nil
230}
231
232// supersededLoss is the conflicted declaration a lost record still names,
233// with the target phase, Dirty and ReturnLane it signed (a landing's
234// declaration signed none: its rollback started the target releasing).
235type supersededLoss struct {
236 in moveLossIntent
237 ref string
238}
239
240// readSuperseded reads the declaration a lost record names. Only a signed,
241// conflicted declaration for this host whose id the claim never took may be
242// superseded; the caller holds its intent to this move and target. A record
243// that names no loss has nothing to supersede. Read-only.
244func readSuperseded(ctx context.Context, store release.Store, pub ed25519.PublicKey, o hostclaim.Observed, rec provision.Record) (*supersededLoss, error) {
245 if rec.Move == nil || rec.Move.Rollback == nil || rec.Move.Rollback.LossRef == "" {
246 return nil, nil
247 }
248 rb := rec.Move.Rollback
249 id := strings.TrimSuffix(strings.TrimPrefix(rb.LossRef, "control/host-loss/"), ".json")
250 if key, err := hostclaim.LossRecordKey(id); err != nil || key != rb.LossRef || rb.TargetPhase != provision.RbTargetLost {
251 return nil, hostresolve.ErrResolve
252 }
253 loss, err := hostclaim.ReadLossRecord(ctx, store, pub, id)
254 if errors.Is(err, hostclaim.ErrLossRecordAbsent) || errors.Is(err, hostclaim.ErrLossRecord) {
255 return nil, hostresolve.ErrResolve
256 }
257 if err != nil {
258 return nil, err
259 }
260 if loss.Record.State != "conflicted" || loss.Record.Host != o.Host || o.Claim == nil || slicesContain(o.Claim.LossRefs, id) {
261 return nil, hostresolve.ErrResolve
262 }
263 old, ref, err := lossIntentOf(loss.Record)
264 if err != nil || ref != rb.LossRef {
265 return nil, hostresolve.ErrResolve
266 }
267 if old.RecordState == moveLossRecordLanding {
268 old.TargetPhase, old.Dirty, old.ReturnLane = provision.RbTargetReleasing, false, false
269 }
270 return &supersededLoss{in: old, ref: ref}, nil
271}
272
273// sameMoveTarget reports two intents for the same move, incarnation,
274// target, regions, record and cut.
275func sameMoveTarget(a, b moveLossIntent) bool {
276 return a.Kind == b.Kind && a.Move == b.Move && a.Incarnation == b.Incarnation && a.Target == b.Target && a.Source == b.Source &&
277 a.TargetRegion == b.TargetRegion && a.SourceRegion == b.SourceRegion && a.RecordKey == b.RecordKey &&
278 a.SrcCovered == b.SrcCovered && a.CutSession == b.CutSession && a.SrcGen == b.SrcGen
279}
280
281// lossIntentAt is the intent the claim o and the record r imply for box,
282// with the declaration a lost record still names. Read-only.
283func lossIntentAt(ctx context.Context, store release.Store, pub ed25519.PublicKey, o hostclaim.Observed, r Record, box string) (moveLossIntent, error) {
284 in, err := claimLossIntent(o.Claim, box)
285 if err != nil {
286 return moveLossIntent{}, err
287 }
288 if !matches(r, o.Claim.Holder) {
289 return moveLossIntent{}, ErrIdentity
290 }
291 _, seq, err := recordSequence(r.Body)
292 if err != nil {
293 return moveLossIntent{}, err
294 }
295 rec, err := provision.DecodeRecord(r.Body)
296 if err != nil {
297 return moveLossIntent{}, ErrIdentity
298 }
299 prior, err := readSuperseded(ctx, store, pub, o, rec)
300 if err != nil {
301 return moveLossIntent{}, err
302 }
303 if err := recordLossIntent(&in, rec, r.ETag, seq, o.Claim.Op, o.ETag, prior); err != nil {
304 return moveLossIntent{}, err
305 }
306 return in, nil
307}
308
309// recordLossIntent is the record's half: the record journals this move for
310// this incarnation, has not committed, and its target still holds what the
311// move needs (§7.10: it owes its stop evidence, or released dirty and has
312// not returned). A landing's rollback id is derived from the claim ETag. A
313// record lost for a conflicted declaration (prior) keeps that declaration's
314// rollback, phase, Dirty and return lane.
315func recordLossIntent(in *moveLossIntent, rec provision.Record, etag string, seq uint64, op hostclaim.Op, claimETag string, prior *supersededLoss) error {
316 m := rec.Move
317 if m == nil || m.ID != in.Move || m.Incarnation != in.Incarnation || !park.PlacementMatches(op.Intent["home_region"], m.To, in.TargetRegion) || park.PlacementRegion(m.From) == "" || m.From == m.To {
318 return hostresolve.ErrResolve
319 }
320 if m.State == provision.MoveCommitted || m.CommittedAt != nil {
321 return moveLossRefusal(provision.RefusedCommitted)
322 }
323 in.SourceRegion, in.RecordETag, in.RecordSeq = park.PlacementRegion(m.From), etag, seq
324 switch m.State {
325 case provision.MoveLanding:
326 if op.Kind != hostclaim.KindLanding || m.Rollback != nil {
327 return hostresolve.ErrResolve
328 }
329 in.RecordState, in.Rollback = moveLossRecordLanding, moveLossRollbackID(op.ID, claimETag, in.Target)
330 return nil
331 case provision.MoveRollingBack:
332 default:
333 return moveLossRefusal("the move's state does not allow a declared loss")
334 }
335 rb := m.Rollback
336 if rb == nil || !provision.ValidRollbackID(rb.ID) {
337 return hostresolve.ErrResolve
338 }
339 phase, dirty := rb.TargetPhase, rb.Dirty
340 if rb.LossRef != "" || prior != nil {
341 if prior == nil || prior.ref != rb.LossRef || rb.TargetPhase != provision.RbTargetLost || prior.in.Rollback != rb.ID || prior.in.Dirty != rb.Dirty || !sameMoveTarget(prior.in, *in) {
342 return hostresolve.ErrResolve
343 }
344 phase, dirty, in.Supersedes = prior.in.TargetPhase, prior.in.Dirty, prior.ref
345 }
346 switch op.Kind {
347 case hostclaim.KindRestoring:
348 if rb.ID != in.Rollback {
349 return hostresolve.ErrResolve
350 }
351 case hostclaim.KindLanding:
352 // The record decided the rollback; the claim has not followed yet.
353 in.Rollback = rb.ID
354 }
355 in.RecordState, in.TargetPhase, in.Dirty = moveLossRecordRollingBack, phase, dirty
356 switch {
357 case phase == provision.RbTargetReleasing:
358 if op.Phase != hostclaim.PhaseReleasing && op.Kind != hostclaim.KindLanding {
359 return hostresolve.ErrResolve
360 }
361 case phase == provision.RbTargetReturning, phase == provision.RbTargetReleased && dirty:
362 if op.Kind != hostclaim.KindRestoring || op.Phase != hostclaim.PhaseReleased {
363 return hostresolve.ErrResolve
364 }
365 in.ReturnLane = true
366 default:
367 return moveLossRefusal(provision.RefusedNothingToDeclare)
368 }
369 if prior != nil && prior.in.ReturnLane != in.ReturnLane {
370 return hostresolve.ErrResolve
371 }
372 return nil
373}
374
375// moveLossConsequences are the shown and actual effects for an intent: the
376// target loses execution (when it holds it) and participation; the regional
377// work it owned goes to its region; nothing about its disks is asserted; the
378// writes and timers since the cut are possibly lost.
379func moveLossConsequences(c *hostclaim.Claim, box string, in moveLossIntent) (hostfence.Consequences, hostclaim.LossConsequences, error) {
380 intent, err := in.encode()
381 if err != nil {
382 return hostfence.Consequences{}, hostclaim.LossConsequences{}, err
383 }
384 shown := hostfence.Consequences{Transition: moveLossTransition, PossibleWriteLoss: true, PossibleTimerLoss: true}
385 actual := hostclaim.LossConsequences{RevokeExecution: c.Op.Exec == box, RemoveParticipation: true, WritesPotentiallyLost: true, TimersPotentiallyLost: true, RecoveryIntent: intent}
386 for _, o := range c.Obligations {
387 if (o.Kind == "park-replica" || o.Kind == "park-generation") && o.Owner == "box:"+box {
388 actual.ReassignRegional = append(actual.ReassignRegional, hostclaim.ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement})
389 shown.Chores = append(shown.Chores, o.Op+"/"+o.ID+"/"+o.Retirement)
390 }
391 }
392 return shown, actual, nil
393}
394
395// moveLossOwner is the moves owner. store is the scoped store of one
396// declaration (or a read-only store for inspection); it is never exposed.
397type moveLossOwner struct {
398 store release.Store
399 pub ed25519.PublicKey
400 now func() time.Time
401}
402
403var _ hostresolve.OperationOwner = (*moveLossOwner)(nil)
404
405func (m *moveLossOwner) clock() time.Time {
406 if m.now != nil {
407 return m.now().UTC()
408 }
409 return time.Now().UTC()
410}
411
412// InspectLoss is read-only. The record must be exactly the observation the
413// service bound into the challenge.
414func (m *moveLossOwner) InspectLoss(ctx context.Context, o hostclaim.Observed, record hostfence.RecordObservation, box string) (hostfence.Consequences, hostclaim.LossConsequences, error) {
415 if o.Claim == nil || o.Claim.Holder == nil {
416 return hostfence.Consequences{}, hostclaim.LossConsequences{}, hostresolve.ErrResolve
417 }
418 if o.Absent || o.ETag == "" || record.Key != o.Claim.Holder.Key || record.CommitState != "present" || record.ETag == "" {
419 return hostfence.Consequences{}, hostclaim.LossConsequences{}, hostresolve.ErrResolve
420 }
421 r, err := ReadRecord(ctx, m.store, record.Key)
422 if err != nil {
423 return hostfence.Consequences{}, hostclaim.LossConsequences{}, err
424 }
425 if r.ETag != record.ETag {
426 return hostfence.Consequences{}, hostclaim.LossConsequences{}, hostclaim.ErrConflict
427 }
428 if moveLocalLossEligible(o.Claim, box) {
429 if !matches(r, o.Claim.Holder) {
430 return hostfence.Consequences{}, hostclaim.LossConsequences{}, ErrIdentity
431 }
432 rec, e := provision.DecodeRecord(r.Body)
433 if e != nil || rec.Move == nil || rec.Move.ID != o.Claim.Op.ID || rec.Move.Incarnation != o.Claim.Holder.Incarnation {
434 return hostfence.Consequences{}, hostclaim.LossConsequences{}, ErrIdentity
435 }
436 return moveLocalLossConsequences(o.Claim, box)
437 }
438 in, err := lossIntentAt(ctx, m.store, m.pub, o, r, box)
439 if err != nil {
440 return hostfence.Consequences{}, hostclaim.LossConsequences{}, err
441 }
442 return moveLossConsequences(o.Claim, box, in)
443}
444
445// ValidateDeclareLost is pure: inside the claim CAS, the signed effects
446// must be exactly what this claim implies for the signed intent, whose
447// claim-derived values must be this claim's.
448func (m *moveLossOwner) ValidateDeclareLost(c hostclaim.Claim, d hostclaim.LostDeclaration) error {
449 if isMoveLocalLoss(d.Consequences) {
450 _, actual, err := moveLocalLossConsequences(&c, d.Box)
451 if err != nil || !reflect.DeepEqual(actual, d.Consequences) {
452 return hostresolve.ErrResolve
453 }
454 return nil
455 }
456 in, err := decodeMoveLossIntent(d.Consequences.RecoveryIntent)
457 if err != nil {
458 return err
459 }
460 want, err := claimLossIntent(&c, d.Box)
461 if err != nil {
462 return err
463 }
464 if in.Move != want.Move || in.Move != d.OpID || in.Incarnation != want.Incarnation || in.Incarnation != d.HolderIncarnation || in.Target != want.Target ||
465 in.Source != want.Source || in.TargetRegion != want.TargetRegion || in.RecordKey != want.RecordKey || in.SrcCovered != want.SrcCovered ||
466 in.CutSession != want.CutSession || in.SrcGen != want.SrcGen || in.SourceRegion == in.TargetRegion {
467 return hostresolve.ErrResolve
468 }
469 if in.Supersedes != "" {
470 // A rebind keeps the rollback the conflicted declaration decided; it
471 // is never recomputed from this claim's ETag.
472 own, err := hostclaim.LossRecordKey(d.LossID)
473 if err != nil || in.Supersedes == own || in.RecordState != moveLossRecordRollingBack {
474 return hostresolve.ErrResolve
475 }
476 for _, id := range c.LossRefs {
477 if key, _ := hostclaim.LossRecordKey(id); key == in.Supersedes {
478 return hostresolve.ErrResolve
479 }
480 }
481 }
482 switch {
483 case c.Op.Kind == hostclaim.KindLanding && in.RecordState == moveLossRecordLanding:
484 if in.Rollback != moveLossRollbackID(c.Op.ID, d.ExpectedClaimETag, d.Box) || in.TargetPhase != "" || in.Dirty || in.ReturnLane {
485 return hostresolve.ErrResolve
486 }
487 case c.Op.Kind == hostclaim.KindLanding:
488 if in.TargetPhase != provision.RbTargetReleasing || in.ReturnLane {
489 return hostresolve.ErrResolve
490 }
491 case c.Op.Kind == hostclaim.KindRestoring && in.RecordState == moveLossRecordRollingBack && in.Rollback == want.Rollback:
492 releasing := c.Op.Phase == hostclaim.PhaseReleasing && in.TargetPhase == provision.RbTargetReleasing && !in.ReturnLane
493 owed := c.Op.Phase == hostclaim.PhaseReleased && in.ReturnLane && in.Dirty &&
494 (in.TargetPhase == provision.RbTargetReturning || in.TargetPhase == provision.RbTargetReleased)
495 if !releasing && !owed {
496 return hostresolve.ErrResolve
497 }
498 default:
499 return hostresolve.ErrResolve
500 }
501 _, actual, err := moveLossConsequences(&c, d.Box, in)
502 if err != nil {
503 return err
504 }
505 got := d.Consequences
506 if got.RecoveryIntent, err = canonicalJSON(got.RecoveryIntent); err != nil {
507 return hostresolve.ErrResolve
508 }
509 if !reflect.DeepEqual(actual, got) {
510 return hostresolve.ErrResolve
511 }
512 return nil
513}
514
515// LossWriteSources freezes every write the declaration may lose: each
516// target session admitted since the cut (its generation in the target
517// region), and the source's cut session when the seal did not cover the
518// source's own replica (the seal in the lost target's region is the only
519// holder of those writes, §7.10). Pure.
520func (m *moveLossOwner) LossWriteSources(_ context.Context, o hostclaim.Observed, actual hostclaim.LossConsequences) ([]hostclaim.WriteSource, error) {
521 if isMoveLocalLoss(actual) {
522 return nil, nil
523 }
524 in, err := decodeMoveLossIntent(actual.RecoveryIntent)
525 if err != nil {
526 return nil, err
527 }
528 c := o.Claim
529 if c == nil || c.Holder == nil || c.Op.ID != in.Move || c.Holder.Incarnation != in.Incarnation {
530 return nil, hostresolve.ErrResolve
531 }
532 h := c.Holder
533 var out []hostclaim.WriteSource
534 seen := map[string]bool{}
535 add := func(s hostclaim.WriteSource) error {
536 if s.Box == "" || (s.Session == 0) != (s.Generation == "") || s.Region == "" {
537 return hostresolve.ErrResolve
538 }
539 key := s.Box + "\x00" + strconv.FormatUint(s.Session, 10)
540 if !seen[key] {
541 seen[key] = true
542 out = append(out, s)
543 }
544 return nil
545 }
546 for _, a := range h.Admissions {
547 if a.Box != in.Target || a.Session <= in.CutSession {
548 continue
549 }
550 if err := add(hostclaim.WriteSource{Reason: "target", Box: a.Box, Generation: a.Generation, Session: a.Session, Region: a.Region, Baseline: MoveLossReplicaBaseline + a.Region + "/" + a.Generation}); err != nil {
551 return nil, err
552 }
553 }
554 if h.WriterBox == in.Target && h.Session > in.CutSession {
555 if err := add(hostclaim.WriteSource{Reason: "target", Box: h.WriterBox, Generation: h.Generation, Session: h.Session, Region: in.TargetRegion, Baseline: MoveLossReplicaBaseline + in.TargetRegion + "/" + h.Generation}); err != nil {
556 return nil, err
557 }
558 }
559 if !in.SrcCovered {
560 if err := add(hostclaim.WriteSource{Reason: "source-cut", Box: in.Source, Generation: in.SrcGen, Session: in.CutSession, Region: in.SourceRegion, Baseline: MoveLossSealBaseline + in.Move}); err != nil {
561 return nil, err
562 }
563 }
564 return out, nil
565}
566
567// lossIntentOf decodes a loss record's intent and holds the record's own
568// identities to it.
569func lossIntentOf(loss hostclaim.LossRecord) (moveLossIntent, string, error) {
570 in, err := decodeMoveLossIntent(loss.ConfirmedConsequences.RecoveryIntent)
571 if err != nil {
572 return moveLossIntent{}, "", err
573 }
574 if loss.OpID != in.Move || loss.Box != in.Target || loss.HolderIncarnation != in.Incarnation {
575 return moveLossIntent{}, "", hostresolve.ErrResolve
576 }
577 ref, err := hostclaim.LossRecordKey(loss.ID)
578 return in, ref, err
579}
580
581// durableLoss rereads the signed loss record and holds it to the hook's
582// copy: the same declaration, the same signed intent.
583func (m *moveLossOwner) durableLoss(ctx context.Context, loss hostclaim.LossRecord) (hostclaim.LossRecord, error) {
584 durable, err := hostclaim.ReadLossRecord(ctx, m.store, m.pub, loss.ID)
585 if err != nil {
586 return hostclaim.LossRecord{}, err
587 }
588 a, err := canonicalJSON(durable.Record.ConfirmedConsequences.RecoveryIntent)
589 if err != nil {
590 return hostclaim.LossRecord{}, err
591 }
592 b, err := canonicalJSON(loss.ConfirmedConsequences.RecoveryIntent)
593 if err != nil || !bytes.Equal(a, b) || durable.Record.Host != loss.Host || durable.Record.ExpectedClaimETag != loss.ExpectedClaimETag {
594 return hostclaim.LossRecord{}, hostresolve.ErrResolve
595 }
596 return durable.Record, nil
597}
598
599// moveLossRecordDone reports a record that already carries this
600// declaration (lost, or released or cleaned with it as the evidence), or a
601// move that has closed. Anything for another move or incarnation refuses.
602func moveLossRecordDone(rec provision.Record, in moveLossIntent, ref string) (lost, released bool, err error) {
603 m := rec.Move
604 if m == nil {
605 if rec.LastMove != nil && rec.LastMove.ID == in.Move {
606 return true, true, nil
607 }
608 return false, false, ErrIdentity
609 }
610 if m.ID != in.Move || m.Incarnation != in.Incarnation {
611 return false, false, ErrIdentity
612 }
613 rb := m.Rollback
614 if rb == nil || rb.LossRef == "" {
615 return false, false, nil
616 }
617 if in.Supersedes != "" && rb.LossRef == in.Supersedes && rb.ID == in.Rollback && rb.TargetPhase == provision.RbTargetLost {
618 return false, false, nil
619 }
620 if rb.ID != in.Rollback || rb.LossRef != ref {
621 return false, false, hostclaim.ErrConflict
622 }
623 if rb.TargetPhase == provision.RbTargetLost {
624 return true, false, nil
625 }
626 if rb.Evidence == ref && !rb.Dirty && (rb.TargetPhase == provision.RbTargetReleased || rb.TargetPhase == provision.RbTargetCleaned) {
627 return true, true, nil
628 }
629 return false, false, hostclaim.ErrWrongState
630}
631
632// prepareMoveLoss is the record's half of the declaration: a landing rolls
633// back for the declared loss, and the rollback's target goes to lost with
634// the loss record as its reference. A rebind moves a lost target's
635// reference from the superseded declaration to this one.
636func prepareMoveLoss(rec *provision.Record, in moveLossIntent, ref string, at time.Time) error {
637 g := provision.MoveGuard{ID: in.Move, Incarnation: in.Incarnation}
638 if in.Supersedes != "" {
639 if in.RecordState != moveLossRecordRollingBack || rec.Move == nil || rec.Move.Rollback == nil || rec.Move.Rollback.ID != in.Rollback {
640 return ErrIdentity
641 }
642 return provision.MoveRollbackLossRebind(g, in.Supersedes, ref, at)(rec)
643 }
644 if in.RecordState == moveLossRecordLanding {
645 if err := provision.MoveRollBack(g, in.Rollback, provision.RollbackDeclaredLost, at)(rec); err != nil {
646 return err
647 }
648 } else if rec.Move == nil || rec.Move.Rollback == nil || rec.Move.Rollback.ID != in.Rollback {
649 return ErrIdentity
650 }
651 return provision.MoveRollbackTargetLost(g, ref, at)(rec)
652}
653
654// PrepareLoss writes the record's decision before the claim CAS, so the
655// move can no longer commit. It never writes the claim, never regresses a
656// record that already carries this declaration, and writes only while the
657// signed loss is prepared, the claim is the declaration's exact observation
658// and the record is the inspected one (ETag and record_seq).
659func (m *moveLossOwner) PrepareLoss(ctx context.Context, o hostclaim.Observed, loss hostclaim.LossRecord) error {
660 if isMoveLocalLoss(loss.ConfirmedConsequences) {
661 return m.localLossHook(ctx, o, loss, false)
662 }
663 in, ref, err := lossIntentOf(loss)
664 if err != nil {
665 return err
666 }
667 if loss.Host != o.Host {
668 return ErrIdentity
669 }
670 r, err := ReadRecord(ctx, m.store, in.RecordKey)
671 if err != nil {
672 return err
673 }
674 rec, err := provision.DecodeRecord(r.Body)
675 if err != nil {
676 return ErrIdentity
677 }
678 if done, _, err := moveLossRecordDone(rec, in, ref); err != nil || done {
679 return err
680 }
681 durable, err := m.durableLoss(ctx, loss)
682 if err != nil {
683 return err
684 }
685 switch durable.State {
686 case "prepared":
687 case "committed":
688 // A concurrent declaration committed; its FinishLoss writes the record.
689 return nil
690 default:
691 return hostclaim.ErrConflict
692 }
693 claim, err := hostclaim.Read(ctx, m.store, loss.Host)
694 if err != nil {
695 return err
696 }
697 if claim.Absent || claim.ETag != durable.ExpectedClaimETag {
698 return hostclaim.ErrConflict
699 }
700 _, seq, err := recordSequence(r.Body)
701 if err != nil {
702 return err
703 }
704 if r.ETag != in.RecordETag || seq != in.RecordSeq || !matches(r, claim.Claim.Holder) {
705 return hostclaim.ErrConflict
706 }
707 if in.Supersedes != "" {
708 // A rebind rereads the superseded declaration: still conflicted,
709 // and the exact intent this claim and record imply.
710 if want, err := lossIntentAt(ctx, m.store, m.pub, claim, r, in.Target); err != nil || !reflect.DeepEqual(want, in) {
711 return hostclaim.ErrConflict
712 }
713 }
714 if err := prepareMoveLoss(&rec, in, ref, durable.FencedAt); err != nil {
715 return err
716 }
717 rec.Move.UpdatedAt = m.clock()
718 body, err := json.Marshal(rec)
719 if err != nil {
720 return err
721 }
722 _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), m.store, in.RecordKey, r.Body, body, r.ETag)
723 return sequenceError(err)
724}
725
726// moveLossRelease is the claim release the signed declaration implies for
727// the holder it names. Every scope and id derives from the signed record.
728func moveLossRelease(in moveLossIntent, loss hostclaim.LossRecord, h *hostclaim.Holder, host string) hostclaim.LostTargetRelease {
729 scope := func(location string) hostclaim.ParkScope {
730 return hostclaim.ParkScope{Location: location, Incarnation: h.Incarnation, EvidenceAt: loss.FencedAt.UTC(), RequestLimit: moveLossRequestLimit}
731 }
732 r := hostclaim.LostTargetRelease{OpID: in.Move, RollbackID: in.Rollback, LossID: loss.ID, Target: in.Target, TargetRegion: in.TargetRegion, SourceRegion: in.SourceRegion,
733 Location: scope(strings.TrimSuffix(h.Location, "/")), Lane: scope(moves.ScopeLocation(host, in.Move)), NewID: moveLossNewID(loss.ID)}
734 if in.ReturnLane {
735 lane := scope(moves.ScopeLocation(host, in.Rollback))
736 r.ReturnLane = &lane
737 }
738 return r
739}
740
741// FinishLoss completes the declaration after its claim CAS committed: the
742// claim releases the lost target (ReleaseLostTarget), then the record's
743// lost target becomes a clean release with the loss as its evidence, which
744// the source's recovery takes. Each step is skipped once durable. A
745// successor holder or operation is never touched: only a record that
746// already shows this declaration complete is success then.
747func (m *moveLossOwner) FinishLoss(ctx context.Context, o hostclaim.Observed, loss hostclaim.LossRecord) error {
748 if isMoveLocalLoss(loss.ConfirmedConsequences) {
749 return m.localLossHook(ctx, o, loss, true)
750 }
751 in, ref, err := lossIntentOf(loss)
752 if err != nil {
753 return err
754 }
755 if loss.Host != o.Host {
756 return ErrIdentity
757 }
758 durable, err := m.durableLoss(ctx, loss)
759 if err != nil {
760 return err
761 }
762 if durable.State != "committed" {
763 return hostclaim.ErrConflict
764 }
765 claim, err := hostclaim.Read(ctx, m.store, loss.Host)
766 if err != nil {
767 return err
768 }
769 ours := !claim.Absent && claim.Claim != nil && claim.Claim.Holder != nil && claim.Claim.Holder.Incarnation == in.Incarnation && claim.Claim.Op.ID == in.Move
770 released := false
771 if ours {
772 c := claim.Claim
773 if !slicesContain(c.LossRefs, loss.ID) {
774 return hostclaim.ErrConflict
775 }
776 if !hostclaim.LostTargetReleased(c, loss.ID) {
777 if _, err := hostclaim.Update(ctx, m.store, claim, hostclaim.ReleaseLostTarget(moveLossRelease(in, durable, c.Holder, loss.Host))); err != nil {
778 return err
779 }
780 }
781 released = true
782 }
783 r, err := ReadRecord(ctx, m.store, in.RecordKey)
784 if err != nil {
785 return err
786 }
787 rec, err := provision.DecodeRecord(r.Body)
788 if err != nil {
789 return ErrIdentity
790 }
791 lost, done, err := moveLossRecordDone(rec, in, ref)
792 if err != nil {
793 return err
794 }
795 if done {
796 return nil
797 }
798 if !lost || !released {
799 return ErrIdentity
800 }
801 if err := provision.MoveRollbackLossCommitted(provision.MoveGuard{ID: in.Move, Incarnation: in.Incarnation})(&rec); err != nil {
802 return err
803 }
804 rec.Move.UpdatedAt = m.clock()
805 body, err := json.Marshal(rec)
806 if err != nil {
807 return err
808 }
809 _, _, err = provision.WriteRecord(provision.WithRecordSequence(ctx), m.store, in.RecordKey, r.Body, body, r.ETag)
810 return sequenceError(err)
811}
812
813func slicesContain(values []string, wanted string) bool {
814 for _, v := range values {
815 if v == wanted {
816 return true
817 }
818 }
819 return false
820}
821
822// errMoveLossScope refuses a write outside one declaration's scope. It wraps
823// ErrContract: the caller asked for something no declaration may do.
824var errMoveLossScope = fmt.Errorf("%w: write outside the declaration's scope", ErrContract)
825