| 1 | package hostresolve |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "reflect" |
| 8 | "slices" |
| 9 | "testing" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/hostclaim" |
| 12 | "github.com/carlosframework/platform/internal/hostfence" |
| 13 | ) |
| 14 | |
| 15 | type plan10LossOwner struct { |
| 16 | service *Service |
| 17 | store *failClaimStore |
| 18 | calls []string |
| 19 | failPrepare, failFinish, mutate bool |
| 20 | sources []hostclaim.WriteSource |
| 21 | } |
| 22 | |
| 23 | func (p *plan10LossOwner) InspectLoss(ctx context.Context, o hostclaim.Observed, record hostfence.RecordObservation, box string) (hostfence.Consequences, hostclaim.LossConsequences, error) { |
| 24 | if o.Claim.Op.Kind != hostclaim.KindMoving || box != "dead" || o.Claim.Op.Intent["target"] != box || record.Key != o.Claim.Holder.Key { |
| 25 | return hostfence.Consequences{}, hostclaim.LossConsequences{}, ErrResolve |
| 26 | } |
| 27 | actual := hostclaim.LossConsequences{RemoveParticipation: true, WritesPotentiallyLost: true, TimersPotentiallyLost: true, RecoveryIntent: json.RawMessage(`{"kind":"test-move-loss"}`)} |
| 28 | if p.mutate { |
| 29 | o.Claim.Op.Intent["target"] = "wrong" |
| 30 | o.Claim.Holder.Stamp++ |
| 31 | } |
| 32 | return hostfence.Consequences{Transition: "test-move-loss", PossibleWriteLoss: true, PossibleTimerLoss: true}, actual, nil |
| 33 | } |
| 34 | func (p *plan10LossOwner) LossWriteSources(context.Context, hostclaim.Observed, hostclaim.LossConsequences) ([]hostclaim.WriteSource, error) { |
| 35 | return p.sources, nil |
| 36 | } |
| 37 | func (p *plan10LossOwner) ValidateDeclareLost(c hostclaim.Claim, d hostclaim.LostDeclaration) error { |
| 38 | if c.Op.Kind != hostclaim.KindMoving || c.Op.Intent["target"] != d.Box || d.Box != "dead" || !d.Consequences.RemoveParticipation { |
| 39 | return ErrResolve |
| 40 | } |
| 41 | return nil |
| 42 | } |
| 43 | func (p *plan10LossOwner) PrepareLoss(ctx context.Context, o hostclaim.Observed, loss hostclaim.LossRecord) error { |
| 44 | p.calls = append(p.calls, "prepare") |
| 45 | roster, err := p.service.Directory.Read(ctx) |
| 46 | if err != nil || roster.Roster.Boxes[loss.Box].State != "quarantined" { |
| 47 | return ErrResolve |
| 48 | } |
| 49 | durable, err := hostclaim.ReadLossRecord(ctx, p.store, p.service.Directory.PublicKey, loss.ID) |
| 50 | if err != nil || durable.Record.State != "prepared" || slices.Contains(o.Claim.LossRefs, loss.ID) { |
| 51 | return ErrResolve |
| 52 | } |
| 53 | if p.failPrepare { |
| 54 | return ErrResolve |
| 55 | } |
| 56 | if p.mutate { |
| 57 | o.Claim.Op.Intent["target"] = "wrong" |
| 58 | loss.ConfirmedConsequences.RecoveryIntent[0] = 'x' |
| 59 | } |
| 60 | return nil |
| 61 | } |
| 62 | func (p *plan10LossOwner) FinishLoss(ctx context.Context, o hostclaim.Observed, loss hostclaim.LossRecord) error { |
| 63 | p.calls = append(p.calls, "finish") |
| 64 | durable, err := hostclaim.ReadLossRecord(ctx, p.store, p.service.Directory.PublicKey, loss.ID) |
| 65 | if err != nil || durable.Record.State != "committed" || !slices.Contains(o.Claim.LossRefs, loss.ID) { |
| 66 | return ErrResolve |
| 67 | } |
| 68 | if p.failFinish { |
| 69 | return ErrResolve |
| 70 | } |
| 71 | if p.mutate { |
| 72 | o.Claim.Op.Intent["target"] = "wrong" |
| 73 | loss.ConfirmedConsequences.RecoveryIntent[0] = 'x' |
| 74 | } |
| 75 | return nil |
| 76 | } |
| 77 | func plan10ResolveMove(t *testing.T) (Service, *plan10LossOwner, hostfence.Submission, string) { |
| 78 | t.Helper() |
| 79 | s, store, now, host := resolveFixture(t) |
| 80 | observed, err := hostclaim.Read(t.Context(), store, host) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | _, err = hostclaim.Update(t.Context(), store, observed, func(c *hostclaim.Claim) error { |
| 85 | c.Op = hostclaim.Op{ID: "move-test", Kind: hostclaim.KindMoving, Phase: hostclaim.PhaseMoving, Exec: "live", Participants: []string{"live", "dead"}, Intent: map[string]string{"source": "live", "target": "dead", "to": "de"}} |
| 86 | return nil |
| 87 | }) |
| 88 | if err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | owner := &plan10LossOwner{store: store} |
| 92 | s.Owners = map[string]OperationOwner{hostclaim.KindMoving: owner} |
| 93 | owner.service = &s |
| 94 | return s, owner, submitted(t, s, now, host), host |
| 95 | } |
| 96 | func TestPlan10MoveOwnerRequiredBeforeAcceptance(t *testing.T) { |
| 97 | s, p, in, host := plan10ResolveMove(t) |
| 98 | s.Owners = nil |
| 99 | before, _ := s.Directory.Read(t.Context()) |
| 100 | if _, err := s.Inspect(t.Context(), "operator", host, "dead"); !errors.Is(err, ErrResolve) { |
| 101 | t.Fatal(err) |
| 102 | } |
| 103 | if _, err := s.Declare(t.Context(), "operator", host, in); !errors.Is(err, ErrResolve) { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | after, _ := s.Directory.Read(t.Context()) |
| 107 | if after.ETag != before.ETag { |
| 108 | t.Fatal("missing owner quarantined") |
| 109 | } |
| 110 | keys, err := p.store.List(t.Context(), "control/host-loss-acceptances/") |
| 111 | if err != nil || len(keys) != 0 { |
| 112 | t.Fatal(keys, err) |
| 113 | } |
| 114 | } |
| 115 | func TestPlan10OwnerHookOrderAndReplay(t *testing.T) { |
| 116 | for _, fault := range []string{"prepare", "claim", "bookkeeping", "finish"} { |
| 117 | t.Run(fault, func(t *testing.T) { |
| 118 | s, p, in, host := plan10ResolveMove(t) |
| 119 | p.failPrepare = fault == "prepare" |
| 120 | p.store.fail = fault == "claim" |
| 121 | p.failFinish = fault == "finish" |
| 122 | bookkeeping := &lossBookkeepingStore{Store: p.store, fail: fault == "bookkeeping"} |
| 123 | s.Directory.Store = bookkeeping |
| 124 | if _, err := s.Declare(t.Context(), "operator", host, in); err == nil { |
| 125 | t.Fatal("fault not reached") |
| 126 | } |
| 127 | p.failPrepare = false |
| 128 | p.store.fail = false |
| 129 | p.failFinish = false |
| 130 | bookkeeping.fail = false |
| 131 | if _, err := s.Declare(t.Context(), "operator", host, in); err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | if p.calls[len(p.calls)-1] != "finish" { |
| 135 | t.Fatal(p.calls) |
| 136 | } |
| 137 | before, _ := hostclaim.Read(t.Context(), p.store, host) |
| 138 | n := len(p.calls) |
| 139 | if _, err := s.Declare(t.Context(), "operator", host, in); err != nil { |
| 140 | t.Fatal(err) |
| 141 | } |
| 142 | after, _ := hostclaim.Read(t.Context(), p.store, host) |
| 143 | if before.ETag != after.ETag || !reflect.DeepEqual(p.calls[n:], []string{"finish"}) { |
| 144 | t.Fatal("retry rewrote claim or re-prepared", p.calls) |
| 145 | } |
| 146 | }) |
| 147 | } |
| 148 | } |
| 149 | func TestPlan10OwnerReceivesDetachedValues(t *testing.T) { |
| 150 | s, p, in, host := plan10ResolveMove(t) |
| 151 | p.mutate = true |
| 152 | after, err := s.Declare(t.Context(), "operator", host, in) |
| 153 | if err != nil { |
| 154 | t.Fatal(err) |
| 155 | } |
| 156 | if after.Claim.Op.Intent["target"] != "dead" { |
| 157 | t.Fatal("callback changed claim") |
| 158 | } |
| 159 | loss, err := hostclaim.ReadLossRecord(t.Context(), p.store, s.Directory.PublicKey, in.Confirmation) |
| 160 | if err != nil || !json.Valid(loss.Record.ConfirmedConsequences.RecoveryIntent) { |
| 161 | t.Fatal(loss, err) |
| 162 | } |
| 163 | } |
| 164 | func TestPlan10AcceptedOwnerUsesOriginalKind(t *testing.T) { |
| 165 | s, p, in, host := plan10ResolveMove(t) |
| 166 | if _, err := s.Declare(t.Context(), "operator", host, in); err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | before, _ := hostclaim.Read(t.Context(), p.store, host) |
| 170 | _, err := hostclaim.Update(t.Context(), p.store, before, func(c *hostclaim.Claim) error { c.Op.Kind = hostclaim.KindReverting; return nil }) |
| 171 | if err != nil { |
| 172 | t.Fatal(err) |
| 173 | } |
| 174 | if _, err = s.Declare(t.Context(), "operator", host, in); err != nil { |
| 175 | t.Fatal("retry selected current owner", err) |
| 176 | } |
| 177 | s.Owners = nil |
| 178 | if _, err = s.Declare(t.Context(), "operator", host, in); !errors.Is(err, ErrResolve) { |
| 179 | t.Fatal("retry bypassed missing owner", err) |
| 180 | } |
| 181 | } |
| 182 | |
| 183 | func TestPlan10LossSourcesFrozenAtAcceptance(t *testing.T) { |
| 184 | s, p, in, host := plan10ResolveMove(t) |
| 185 | p.sources = []hostclaim.WriteSource{{Reason: "target", Box: "dead", Session: 12, Generation: "generation-12", Region: "de", Baseline: "generation-12"}} |
| 186 | p.store.fail = true |
| 187 | if _, err := s.Declare(t.Context(), "operator", host, in); err == nil { |
| 188 | t.Fatal("fault not reached") |
| 189 | } |
| 190 | loss, err := hostclaim.ReadLossRecord(t.Context(), p.store, s.Directory.PublicKey, in.Confirmation) |
| 191 | if err != nil || !reflect.DeepEqual(loss.Record.WriteSources, p.sources) { |
| 192 | t.Fatal(loss, err) |
| 193 | } |
| 194 | p.sources[0].Session = 99 |
| 195 | p.store.fail = false |
| 196 | if _, err = s.Declare(t.Context(), "operator", host, in); err != nil { |
| 197 | t.Fatal(err) |
| 198 | } |
| 199 | loss, err = hostclaim.ReadLossRecord(t.Context(), p.store, s.Directory.PublicKey, in.Confirmation) |
| 200 | if err != nil || len(loss.Record.WriteSources) != 1 || loss.Record.WriteSources[0].Session != 12 { |
| 201 | t.Fatal("retry recomputed loss sources", loss, err) |
| 202 | } |
| 203 | } |
| 204 | func TestPlan10BadSourcesRefuseBeforeQuarantine(t *testing.T) { |
| 205 | s, p, in, host := plan10ResolveMove(t) |
| 206 | p.sources = []hostclaim.WriteSource{{Reason: "target", Box: "dead"}} |
| 207 | before, _ := s.Directory.Read(t.Context()) |
| 208 | if _, err := s.Declare(t.Context(), "operator", host, in); err == nil { |
| 209 | t.Fatal("invalid frozen sources accepted") |
| 210 | } |
| 211 | after, _ := s.Directory.Read(t.Context()) |
| 212 | if before.ETag != after.ETag || len(p.calls) != 0 { |
| 213 | t.Fatal("invalid sources changed authority") |
| 214 | } |
| 215 | } |
| 216 | |
amadan