| 1 | //go:build unix |
| 2 | |
| 3 | package hostbox |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/ed25519" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "github.com/carlosframework/platform/internal/genselect" |
| 11 | "github.com/carlosframework/platform/internal/hostclaim" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | "testing" |
| 14 | "time" |
| 15 | ) |
| 16 | |
| 17 | type plan12LossOwner struct{ calls int } |
| 18 | |
| 19 | func (p *plan12LossOwner) ValidateLossRecovery(c hostclaim.Claim, l hostclaim.LossRecord, a hostclaim.Admission) error { |
| 20 | p.calls++ |
| 21 | if c.Op.Kind != hostclaim.KindRestoring || l.OpID != c.Op.ID || a.Box != "box" { |
| 22 | return hostclaim.ErrLossRecord |
| 23 | } |
| 24 | return nil |
| 25 | } |
| 26 | func TestPlan12LossAdmission(t *testing.T) { |
| 27 | plan12LossAdmissionCases(t, []string{"ok", "source-cut", "covered-source", "no-source", "clean-return", "sealed", "missing-owner", "missing-source", "wrong-source", "wrong-op", "wrong-signature", "stale", "restore-mismatch", "bypass"}) |
| 28 | } |
| 29 | func TestLossBeforeTargetAdmissionRecovers(t *testing.T) { |
| 30 | plan12LossAdmissionCases(t, []string{"source-cut", "covered-source", "no-source"}) |
| 31 | } |
| 32 | func plan12LossAdmissionCases(t *testing.T, modes []string) { |
| 33 | for _, mode := range modes { |
| 34 | t.Run(mode, func(t *testing.T) { |
| 35 | _, store, o := workFixture(t) |
| 36 | var err error |
| 37 | o, err = hostclaim.Update(t.Context(), store, o, func(c *hostclaim.Claim) error { |
| 38 | c.Op = hostclaim.Op{Kind: hostclaim.KindRestoring, ID: "op", Phase: hostclaim.PhaseProbing, Exec: "box", Participants: []string{"box"}, Intent: map[string]string{"target": "target"}, Journal: map[string]string{"loss.loss_1": `{"kind":"move-target-lost","target":"target"}`}} |
| 39 | c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = 2, "target-gen", "target" |
| 40 | if mode == "source-cut" || mode == "covered-source" || mode == "no-source" { |
| 41 | c.Holder.WriterBox = "box" |
| 42 | } |
| 43 | if mode == "no-source" { |
| 44 | c.Holder.Session, c.Holder.Generation, c.Holder.WriterBox = 0, "", "" |
| 45 | } |
| 46 | c.LossRefs = []string{"loss_1"} |
| 47 | return nil |
| 48 | }) |
| 49 | if err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | pub, priv, err := ed25519.GenerateKey(nil) |
| 53 | if err != nil { |
| 54 | t.Fatal(err) |
| 55 | } |
| 56 | now := time.Now().UTC() |
| 57 | r := hostclaim.LossRecord{V: 1, ID: "loss_1", State: "prepared", Actor: "operator", AcceptedAt: now, Host: o.Host, HolderIncarnation: o.Claim.Holder.Incarnation, HolderStamp: o.Claim.Holder.Stamp, HolderSession: 2, ExpectedClaimGen: o.Claim.Gen, ExpectedClaimETag: o.ETag, OpID: "op", OpKind: hostclaim.KindLanding, OpPhase: "probing", Box: "target", Enrollment: "enroll", QuarantineRevision: 1, |
| 58 | CloudObservation: hostclaim.LossCloudObservation{ObservedAt: now, Account: "123", Region: "eu-west-1", InstanceID: "i-1", State: "stopped", RequestID: "request"}, OperatorAttestation: hostclaim.LossOperatorAttestation{MaintainComputeFence: true, ColdQuarantineStartOnly: true}, FencedAt: now, |
| 59 | ConfirmedConsequences: hostclaim.LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: json.RawMessage(`{"kind":"move-target-lost"}`)}, Timers: hostclaim.TimerLoss{Knowledge: hostclaim.TimersUnknown}, WriteSources: []hostclaim.WriteSource{{Reason: "target", Box: "target", Session: 2, Generation: "target-gen", Region: "uk", Baseline: "target-snapshot"}}} |
| 60 | if mode == "source-cut" { |
| 61 | r.WriteSources[0].Reason, r.WriteSources[0].Box = "source-cut", "box" |
| 62 | } |
| 63 | if mode == "no-source" || mode == "covered-source" { |
| 64 | r.WriteSources = nil |
| 65 | r.HolderSession = 0 |
| 66 | } |
| 67 | if mode == "wrong-op" { |
| 68 | r.OpID = "different" |
| 69 | } |
| 70 | created, err := hostclaim.CreateLossRecord(t.Context(), store, release.RootSigner(priv), r) |
| 71 | if err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | r.State, r.ResultClaimGen = "committed", o.Claim.Gen |
| 75 | if _, err = hostclaim.ReplaceLossRecord(t.Context(), store, release.RootSigner(priv), created, r); err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | owner := &plan12LossOwner{} |
| 79 | w := Writer{Claims: store, Box: "box", Region: "ie", AdmissionEvidence: func(_ context.Context, obs hostclaim.Observed, admission hostclaim.Admission, baseline *genselect.Plan) error { |
| 80 | if baseline == nil || baseline.MaxTXID != 3 { |
| 81 | t.Fatal("restore baseline changed", baseline) |
| 82 | } |
| 83 | want := uint64(7) |
| 84 | if mode == "no-source" || mode == "covered-source" || mode == "wrong-source" || hostclaim.OperationLossConsumed(obs.Claim, "loss_1") { |
| 85 | want = 3 |
| 86 | } |
| 87 | if admission.RemoteMaxTXID != want { |
| 88 | t.Fatalf("consumer maximum=%d want=%d", admission.RemoteMaxTXID, want) |
| 89 | } |
| 90 | return nil |
| 91 | }, Loss: &LossAdmission{PublicKey: pub, Owner: owner}} |
| 92 | w.Loss.Sources = func(_ context.Context, _ hostclaim.Observed, r hostclaim.LossRecord, _ hostclaim.Admission, _ *genselect.Plan) ([]hostclaim.ValidatedWriteSource, error) { |
| 93 | if mode == "no-source" || mode == "covered-source" { |
| 94 | return nil, nil |
| 95 | } |
| 96 | f := r.WriteSources[0] |
| 97 | v := hostclaim.ValidatedWriteSource{Reason: f.Reason, Box: f.Box, Session: f.Session, Generation: f.Generation, Region: f.Region, Baseline: f.Baseline, RemoteMaxTXID: 7} |
| 98 | if mode == "wrong-source" { |
| 99 | v.Session++ |
| 100 | } |
| 101 | return []hostclaim.ValidatedWriteSource{v}, nil |
| 102 | } |
| 103 | if mode == "missing-owner" { |
| 104 | w.Loss.Owner = nil |
| 105 | } |
| 106 | if mode == "missing-source" { |
| 107 | w.Loss.Sources = nil |
| 108 | } |
| 109 | if mode == "wrong-signature" { |
| 110 | w.Loss.PublicKey, _, _ = ed25519.GenerateKey(nil) |
| 111 | } |
| 112 | in := hostclaim.Admission{ObservedOpID: "op", ObservedPhase: hostclaim.PhaseProbing, ObservedSession: 2, ObservedGeneration: "target-gen", Box: "box", Mode: "cold", NewGeneration: "returned", Region: "ie", PredecessorRegion: "uk", RemoteMaxTXID: 3} |
| 113 | if mode == "no-source" { |
| 114 | in.ObservedSession, in.ObservedGeneration = 0, "" |
| 115 | } |
| 116 | if mode == "clean-return" || mode == "sealed" { |
| 117 | in.Coverage = mode |
| 118 | } |
| 119 | if mode == "stale" { |
| 120 | _, err = hostclaim.Update(t.Context(), store, o, func(c *hostclaim.Claim) error { return nil }) |
| 121 | if err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | } |
| 125 | if mode == "restore-mismatch" { |
| 126 | in.RemoteMaxTXID = 4 |
| 127 | } |
| 128 | var admitted hostclaim.Observed |
| 129 | if mode == "bypass" { |
| 130 | admitted, err = hostclaim.Update(t.Context(), store, o, hostclaim.AdmitWriter(in)) |
| 131 | } else { |
| 132 | admitted, err = w.Admit(t.Context(), o, in, restorePlan(o.Claim.Holder.Location, "", 3)) |
| 133 | } |
| 134 | if mode != "ok" && mode != "source-cut" && mode != "covered-source" && mode != "no-source" { |
| 135 | if err == nil { |
| 136 | t.Fatal("accepted", mode) |
| 137 | } |
| 138 | if mode == "stale" && !errors.Is(err, hostclaim.ErrConflict) { |
| 139 | t.Fatal(err) |
| 140 | } |
| 141 | return |
| 142 | } |
| 143 | if err != nil { |
| 144 | t.Fatal(err) |
| 145 | } |
| 146 | a := admitted.Claim.Holder.Admissions[0] |
| 147 | if a.LossID != "loss_1" || a.Coverage != "" || (mode != "no-source" && mode != "covered-source" && (a.WritesLost == nil || a.WritesLost.Session != 2 || a.WritesLost.RemoteMaxTXID != 7)) || owner.calls != 1 { |
| 148 | t.Fatalf("%+v owner=%d", a, owner.calls) |
| 149 | } |
| 150 | if mode == "covered-source" && (a.WritesLost == nil || a.WritesLost.RemoteMaxTXID != 3) { |
| 151 | t.Fatalf("restore maximum lost: %+v", a) |
| 152 | } |
| 153 | if mode == "no-source" && a.WritesLost != nil { |
| 154 | t.Fatalf("invented predecessor: %+v", a) |
| 155 | } |
| 156 | // Restart loses the writer object, but not the claim's consumption marker. |
| 157 | restarted := Writer{Claims: store, Box: "box", Region: "ie", AdmissionEvidence: w.AdmissionEvidence} |
| 158 | fresh, err := hostclaim.Read(t.Context(), store, o.Host) |
| 159 | if err != nil { |
| 160 | t.Fatal(err) |
| 161 | } |
| 162 | in.ObservedSession, in.ObservedGeneration, in.NewGeneration = fresh.Claim.Holder.Session, fresh.Claim.Holder.Generation, "second-cold" |
| 163 | in.PredecessorRegion = "ie" |
| 164 | in.LossID = "loss_1" |
| 165 | again, err := restarted.Admit(t.Context(), fresh, in, &genselect.Plan{MaxTXID: 3}) |
| 166 | if err != nil { |
| 167 | t.Fatal("second cold admission after restart", err) |
| 168 | } |
| 169 | if again.Claim.Holder.Admissions[0].LossID != "loss_1" || again.Claim.Holder.Admissions[len(again.Claim.Holder.Admissions)-1].LossID != "" { |
| 170 | t.Fatal("loss consumed again") |
| 171 | } |
| 172 | |
| 173 | }) |
| 174 | } |
| 175 | } |
| 176 | |
amadan