| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/ed25519" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "github.com/carlosframework/platform/internal/genselect" |
| 9 | "github.com/carlosframework/platform/internal/hostbox" |
| 10 | "github.com/carlosframework/platform/internal/hostclaim" |
| 11 | "github.com/carlosframework/platform/internal/moves" |
| 12 | "github.com/carlosframework/platform/internal/provision" |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | "github.com/carlosframework/platform/internal/termfence" |
| 15 | "github.com/superfly/ltx" |
| 16 | "strings" |
| 17 | "testing" |
| 18 | ) |
| 19 | |
| 20 | type plan12BumpFence struct { |
| 21 | c7ReplayAuthority |
| 22 | bump func() |
| 23 | reject bool |
| 24 | } |
| 25 | |
| 26 | func (a *plan12BumpFence) ValidateFence(ctx context.Context, f termfence.Record) (hostclaim.FenceBinding, error) { |
| 27 | if a.reject { |
| 28 | return hostclaim.FenceBinding{}, hostclaim.ErrFence |
| 29 | } |
| 30 | if a.bump != nil { |
| 31 | fn := a.bump |
| 32 | a.bump = nil |
| 33 | fn() |
| 34 | } |
| 35 | return a.c7ReplayAuthority.ValidateFence(ctx, f) |
| 36 | } |
| 37 | func TestPlan12RecoveryFenceGrant(t *testing.T) { |
| 38 | for _, mode := range []string{"ok", "admitted", "installed", "from-landing", "from-restoring", "record-released", "changed-sequence", "dirty", "dirty-unregistered", "escaped", "wrong-ref"} { |
| 39 | t.Run(mode, func(t *testing.T) { |
| 40 | w := newC7World(t, func(c *hostclaim.Claim, r *provision.Record, a *AppAdmission) { |
| 41 | c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindLanding, hostclaim.PhaseProbing, "source" |
| 42 | c.Op.Intent["source"], c.Op.Intent["target"], c.Op.Intent["to"], c.Op.Intent["target_enrollment"] = "target", "source", "ie", strings.Repeat("a", 32) |
| 43 | c.Op.Participants = []string{"target", "source"} |
| 44 | c.Obligations = nil |
| 45 | c.Op.Journal = map[string]string{hostclaim.JournalSourceStamp: "1", hostclaim.JournalSrcGen: "src"} |
| 46 | r.Region = "uk" |
| 47 | r.Move.From, r.Move.To, r.Move.State = "uk", "ie", provision.MoveRollingBack |
| 48 | at := r.CreatedAt |
| 49 | r.Move.Rollback = &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", Reason: provision.RollbackTargetFenced, Evidence: "control/box-fences/" + strings.Repeat("a", 32) + ".json", RequestedAt: &at, TargetPhase: provision.RbTargetLost, LossRef: "control/box-fences/" + strings.Repeat("a", 32) + ".json", LostAt: &at, SourcePhase: provision.RbSourceWaiting, Dirty: mode == "dirty" || mode == "dirty-unregistered"} |
| 50 | if mode == "dirty-unregistered" { |
| 51 | // The registration gone: the grant refuses before it |
| 52 | // verifies the fence, so the refusal is not verified-dirty. |
| 53 | delete(a.Registrations, c.Op.ID) |
| 54 | } |
| 55 | if mode == "admitted" || mode == "installed" { |
| 56 | c.Holder.Session = 7 |
| 57 | c.Holder.Stamp = 7 |
| 58 | c.Holder.Generation = "target-gen" |
| 59 | c.Holder.WriterBox = "source" |
| 60 | c.Holder.Admissions = []hostclaim.AdmissionRecord{{Session: 7, Box: "source", Generation: "target-gen", Mode: "cold", Region: "ie"}} |
| 61 | } |
| 62 | if mode == "installed" { |
| 63 | c.Inventory = map[string]hostclaim.InventoryEntry{"source": {Layout: "flat", Paths: map[string]uint64{"app.db": 7}}} |
| 64 | } |
| 65 | if mode == "from-restoring" || mode == "record-released" { |
| 66 | c.Op.Kind, c.Op.Phase = hostclaim.KindRestoring, hostclaim.PhaseReleasing |
| 67 | c.Op.Intent["rollback"] = r.Move.Rollback.ID |
| 68 | c.Op.Journal[hostclaim.JournalTargetStamp] = "7" |
| 69 | r.Move.Rollback.TargetPhase = provision.RbTargetReleasing |
| 70 | r.Move.Rollback.LossRef = "" |
| 71 | r.Move.Rollback.LostAt = nil |
| 72 | r.Move.Rollback.Reason = provision.RollbackLandingDeadline |
| 73 | } |
| 74 | if mode == "record-released" { |
| 75 | r.Move.Rollback.TargetPhase = provision.RbTargetReleased |
| 76 | r.Move.Rollback.Evidence = moves.StoppedKey(moves.ReportPrefix("bab", "app", c.Host, c.Op.ID), "target") |
| 77 | } |
| 78 | if mode == "from-landing" { |
| 79 | r.Region = "ie" |
| 80 | r.Move.State = provision.MoveLanding |
| 81 | r.Move.Rollback = nil |
| 82 | } |
| 83 | }) |
| 84 | if mode == "record-released" { |
| 85 | c := w.claim(t).Claim |
| 86 | report := moves.StoppedReport{V: 1, Role: "target", Box: "source", Host: w.host, OpID: c.Op.ID, Incarnation: c.Holder.Incarnation, CheckedAt: w.now, Steps: map[string]moves.Check{moves.StepTree: {OK: true}, moves.StepStray: {OK: true}, moves.StepFDs: {OK: true}, moves.StepLease: {OK: true}, moves.StepReplica: {OK: true}}} |
| 87 | body, _ := json.Marshal(report) |
| 88 | if _, err := w.raw.Put(t.Context(), moves.StoppedKey(moves.ReportPrefix("bab", "app", w.host, c.Op.ID), "target"), body, ""); err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | } |
| 92 | authority := &plan12BumpFence{c7ReplayAuthority: w.authority} |
| 93 | if mode == "changed-sequence" { |
| 94 | authority.bump = func() { |
| 95 | r, e := ReadRecord(t.Context(), w.raw, w.key) |
| 96 | if e != nil { |
| 97 | t.Fatal(e) |
| 98 | } |
| 99 | if _, _, e = provision.WriteRecord(provision.WithRecordSequence(t.Context()), w.raw, w.key, r.Body, r.Body, r.ETag); e != nil { |
| 100 | t.Fatal(e) |
| 101 | } |
| 102 | } |
| 103 | } |
| 104 | ref := w.ref |
| 105 | if mode == "wrong-ref" { |
| 106 | ref.SHA256 = strings.Repeat("e", 64) |
| 107 | } |
| 108 | var escaped *MoveRecoveryGrant |
| 109 | var out FencedTargetOutcome |
| 110 | run := func() error { |
| 111 | return w.control.RunRecovery(t.Context(), func(ctx context.Context) error { |
| 112 | return runMoveRecovery(ctx, w.claim(t), func(ctx context.Context, g *MoveRecoveryGrant) error { |
| 113 | if mode == "escaped" { |
| 114 | escaped = g |
| 115 | return nil |
| 116 | } |
| 117 | var err error |
| 118 | out, err = g.ReleaseFencedTarget(ctx, ref) |
| 119 | return err |
| 120 | }, authority) |
| 121 | }) |
| 122 | } |
| 123 | err := run() |
| 124 | if mode == "escaped" { |
| 125 | out, err = escaped.ReleaseFencedTarget(t.Context(), ref) |
| 126 | } |
| 127 | // Only a dirty target behind a verified fence is verified-dirty |
| 128 | // (Task 5 fix round 1, item 1). |
| 129 | if (mode == "dirty") != out.VerifiedDirty || (mode == "dirty" || mode == "dirty-unregistered") && !errors.Is(err, ErrPending) { |
| 130 | t.Fatalf("%s: verified-dirty %v, err %v", mode, out.VerifiedDirty, err) |
| 131 | } |
| 132 | if mode != "ok" && mode != "admitted" && mode != "installed" && mode != "from-landing" && mode != "from-restoring" && mode != "record-released" { |
| 133 | if err == nil { |
| 134 | t.Fatal("accepted", mode) |
| 135 | } |
| 136 | if mode == "changed-sequence" && !errors.Is(err, release.ErrCASConflict) { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | return |
| 140 | } |
| 141 | if err != nil { |
| 142 | t.Fatal(err) |
| 143 | } |
| 144 | // The outcome carries the step's own write (Task 5 fix round 1, |
| 145 | // item 2): the record's transition, or the claim's rollback. |
| 146 | wantEvent := map[string]string{"from-landing": "rolling_back", "from-restoring": "declared_lost"}[mode] |
| 147 | if got := ""; out.Transition != nil || wantEvent != "" { |
| 148 | if out.Transition != nil { |
| 149 | got = out.Transition.Event |
| 150 | } |
| 151 | if got != wantEvent { |
| 152 | t.Fatalf("%s: the record's transition %q, want %q", mode, got, wantEvent) |
| 153 | } |
| 154 | } |
| 155 | if wantRollback := mode == "ok" || mode == "admitted" || mode == "installed"; out.ClaimRolledBack != wantRollback { |
| 156 | t.Fatalf("%s: claim rollback %v, want %v", mode, out.ClaimRolledBack, wantRollback) |
| 157 | } |
| 158 | if mode == "from-landing" || mode == "from-restoring" { |
| 159 | if mode == "from-landing" && w.claim(t).Claim.Op.Kind != hostclaim.KindLanding { |
| 160 | t.Fatal("record-first pass changed claim") |
| 161 | } |
| 162 | if err := run(); err != nil { |
| 163 | t.Fatal("claim pass", err) |
| 164 | } |
| 165 | if out.Transition != nil || out.ClaimRolledBack != (mode == "from-landing") { |
| 166 | t.Fatalf("%s: the claim pass's outcome %+v", mode, out) |
| 167 | } |
| 168 | } |
| 169 | o := w.claim(t) |
| 170 | if o.Claim.Op.Phase != hostclaim.PhaseReleased || len(o.Claim.Op.Participants) != 1 || len(o.Claim.Obligations) != func() int { |
| 171 | if mode == "installed" { |
| 172 | return 3 |
| 173 | } |
| 174 | return 2 |
| 175 | }() { |
| 176 | t.Fatal(o.Claim) |
| 177 | } |
| 178 | if len(o.Claim.Lost) != 1 || len(o.Claim.LossRefs) != 1 || len(o.Claim.EvidenceRefs) != 1 || o.Claim.EvidenceRefs[0].Reported { |
| 179 | t.Fatal("missing pending automatic loss", o.Claim) |
| 180 | } |
| 181 | if mode == "admitted" || mode == "installed" { |
| 182 | l := o.Claim.EvidenceRefs[0] |
| 183 | if l.Session != 7 || l.Generation != "target-gen" || l.LastAdmission == nil || l.LastAdmission.Covered { |
| 184 | t.Fatal("lost writer provenance", l) |
| 185 | } |
| 186 | } |
| 187 | // A retired target or witness no longer validates against today's roster. |
| 188 | authority.reject = true |
| 189 | before, _ := json.Marshal(o.Claim.Obligations) |
| 190 | if err := run(); err != nil { |
| 191 | t.Fatal("replay", err) |
| 192 | } |
| 193 | after, _ := json.Marshal(w.claim(t).Claim.Obligations) |
| 194 | if string(before) != string(after) { |
| 195 | t.Fatal("requeued parks") |
| 196 | } |
| 197 | rec, err := provision.GetRecord(t.Context(), w.raw, "bab", "app", w.host) |
| 198 | if err != nil || rec.Move.Rollback.TargetPhase != provision.RbTargetReleased || (mode != "record-released" && rec.Move.Rollback.Evidence != ref.Key) { |
| 199 | t.Fatal(rec, err) |
| 200 | } |
| 201 | if err := run(); err != nil { |
| 202 | t.Fatal("completed replay", err) |
| 203 | } |
| 204 | if mode == "from-restoring" && rec.Move.Rollback.Reason != provision.RollbackLandingDeadline { |
| 205 | t.Fatal("rewrote first reason") |
| 206 | } |
| 207 | act := func(fn func(context.Context, *MoveRecoveryGrant) error) error { |
| 208 | return w.control.RunRecovery(t.Context(), func(ctx context.Context) error { return runMoveRecovery(ctx, w.claim(t), fn, authority) }) |
| 209 | } |
| 210 | if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.StampLostMoveTargetParked(ctx) }); err == nil { |
| 211 | t.Fatal("stamped with parks owed") |
| 212 | } |
| 213 | current := w.claim(t) |
| 214 | for _, ob := range current.Claim.Obligations { |
| 215 | var e error |
| 216 | current, e = hostclaim.Update(t.Context(), w.raw, current, hostclaim.CompleteObligation(ob.Op, ob.ID)) |
| 217 | if e != nil { |
| 218 | t.Fatal(e) |
| 219 | } |
| 220 | } |
| 221 | if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.StampLostMoveTargetParked(ctx) }); err != nil { |
| 222 | t.Fatal("stamp", err) |
| 223 | } |
| 224 | _, stampTag, err := provision.GetRecordETag(t.Context(), w.raw, "bab", "app", w.host) |
| 225 | if err != nil { |
| 226 | t.Fatal(err) |
| 227 | } |
| 228 | claimTag := w.claim(t).ETag |
| 229 | if err := run(); err != nil { |
| 230 | t.Fatal("release replay after parked stamp", err) |
| 231 | } |
| 232 | _, replayTag, err := provision.GetRecordETag(t.Context(), w.raw, "bab", "app", w.host) |
| 233 | if err != nil || replayTag != stampTag || w.claim(t).ETag != claimTag { |
| 234 | t.Fatal("parked replay wrote state", err) |
| 235 | } |
| 236 | if mode == "admitted" { |
| 237 | plan12GrantLossMaximum(t, w) |
| 238 | } |
| 239 | // Source opening is owned by the source; seed its completed evidence. |
| 240 | current = w.claim(t) |
| 241 | if _, err := hostclaim.Update(t.Context(), w.raw, current, func(c *hostclaim.Claim) error { |
| 242 | c.Op.Phase = hostclaim.PhaseOpened |
| 243 | c.Op.Exec = c.Op.Intent["source"] |
| 244 | return nil |
| 245 | }); err != nil { |
| 246 | t.Fatal(err) |
| 247 | } |
| 248 | rr, e := ReadRecord(t.Context(), w.raw, w.key) |
| 249 | if e != nil { |
| 250 | t.Fatal(e) |
| 251 | } |
| 252 | decoded, e := provision.DecodeRecord(rr.Body) |
| 253 | if e != nil { |
| 254 | t.Fatal(e) |
| 255 | } |
| 256 | rec = &decoded |
| 257 | rec.Move.Rollback.OpenedAt, rec.Move.Rollback.RecoveredAt = &w.now, &w.now |
| 258 | body, _ := json.Marshal(rec) |
| 259 | if _, _, e = provision.WriteRecord(provision.WithRecordSequence(t.Context()), w.raw, w.key, rr.Body, body, rr.ETag); e != nil { |
| 260 | t.Fatal(e) |
| 261 | } |
| 262 | if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { |
| 263 | return g.Record(ctx, RecoveryMoveRolledBackClose) |
| 264 | }); err != nil { |
| 265 | t.Fatal("close", err) |
| 266 | } |
| 267 | if err := act(func(ctx context.Context, g *MoveRecoveryGrant) error { return g.Claim(ctx, RecoveryMoveRelease) }); err != nil { |
| 268 | t.Fatal("release", err) |
| 269 | } |
| 270 | afterClose := w.claim(t).Claim |
| 271 | if afterClose.Op.Kind != hostclaim.KindServing || len(afterClose.EvidenceRefs) != 1 || len(afterClose.LossRefs) != 1 { |
| 272 | t.Fatal("loss evidence erased", afterClose) |
| 273 | } |
| 274 | // Reporting works after operation replacement and needs no live roster. |
| 275 | pub, priv, e := ed25519.GenerateKey(nil) |
| 276 | if e != nil { |
| 277 | t.Fatal(e) |
| 278 | } |
| 279 | w.control.pub = pub |
| 280 | if err := w.control.RunRecovery(t.Context(), func(ctx context.Context) error { |
| 281 | return w.control.ReportAutomaticLosses(ctx, release.RootSigner(priv), w.host) |
| 282 | }); err != nil { |
| 283 | t.Fatal("report", err) |
| 284 | } |
| 285 | l := w.claim(t).Claim.EvidenceRefs[0] |
| 286 | reported, e := hostclaim.ReadLossRecord(t.Context(), w.raw, pub, l.ID) |
| 287 | if e != nil || reported.Record.State != "committed" || !reported.Record.AlertsPending || reported.Record.Automatic == nil || reported.Record.Automatic.Fence != ref || !l.Reported { |
| 288 | t.Fatal("durable reporting", reported, e) |
| 289 | } |
| 290 | if err := w.control.RunRecovery(t.Context(), func(ctx context.Context) error { |
| 291 | return w.control.ReportAutomaticLosses(ctx, release.RootSigner(priv), w.host) |
| 292 | }); err != nil { |
| 293 | t.Fatal("report replay", err) |
| 294 | } |
| 295 | |
| 296 | }) |
| 297 | } |
| 298 | } |
| 299 | |
| 300 | func plan12GrantLossMaximum(t *testing.T, w *c7World) { |
| 301 | t.Helper() |
| 302 | o := w.claim(t) |
| 303 | source := o.Claim.Op.Intent["source"] |
| 304 | o, err := hostclaim.Update(t.Context(), w.raw, o, hostclaim.ReturnToSource(o.Claim.Op.ID, source, false, hostclaim.ReturnFence{})) |
| 305 | if err != nil { |
| 306 | t.Fatal(err) |
| 307 | } |
| 308 | o, err = hostclaim.Update(t.Context(), w.raw, o, hostclaim.SetRestoringPhase(o.Claim.Op.ID, hostclaim.PhaseClaimed, hostclaim.PhaseProbing)) |
| 309 | if err != nil { |
| 310 | t.Fatal(err) |
| 311 | } |
| 312 | in := hostclaim.Admission{ObservedOpID: o.Claim.Op.ID, ObservedPhase: o.Claim.Op.Phase, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: source, Mode: "cold", NewGeneration: "recovered", Region: "uk", PredecessorRegion: "ie", RemoteMaxTXID: 3} |
| 313 | checked := false |
| 314 | writer := hostbox.Writer{Claims: w.raw, Box: source, Region: "uk", AdmissionEvidence: func(_ context.Context, _ hostclaim.Observed, a hostclaim.Admission, p *genselect.Plan) error { |
| 315 | checked = true |
| 316 | if a.RemoteMaxTXID != 7 || p.MaxTXID != 3 { |
| 317 | t.Fatalf("loss=%d restore=%d", a.RemoteMaxTXID, p.MaxTXID) |
| 318 | } |
| 319 | return nil |
| 320 | }, Loss: &hostbox.LossAdmission{Sources: func(_ context.Context, _ hostclaim.Observed, l hostclaim.LossRecord, _ hostclaim.Admission, _ *genselect.Plan) ([]hostclaim.ValidatedWriteSource, error) { |
| 321 | if len(l.WriteSources) != 1 { |
| 322 | t.Fatalf("sources %+v", l.WriteSources) |
| 323 | } |
| 324 | s := l.WriteSources[0] |
| 325 | return []hostclaim.ValidatedWriteSource{{Reason: s.Reason, Box: s.Box, Session: s.Session, Generation: s.Generation, Region: s.Region, Baseline: s.Baseline, RemoteMaxTXID: 7}}, nil |
| 326 | }}} |
| 327 | // The restore plan's selected files reach its maximum |
| 328 | // (genselect.GenerationMax, moves Task 4). |
| 329 | root := genselect.File{Key: strings.TrimSuffix(o.Claim.Holder.Location, "/") + "/g-src/0009/" + ltx.FormatFilename(1, 3), VersionID: "v1", Level: 9, MinTXID: 1, MaxTXID: 3, Size: 100} |
| 330 | admitted, err := writer.Admit(t.Context(), o, in, &genselect.Plan{Generation: "src", Files: []genselect.File{root}, Root: &root, MaxTXID: 3}) |
| 331 | if err != nil { |
| 332 | t.Fatal(err) |
| 333 | } |
| 334 | records := admitted.Claim.Holder.Admissions |
| 335 | if !checked || records[len(records)-1].WritesLost.RemoteMaxTXID != 7 { |
| 336 | t.Fatal("fenced maximum lost") |
| 337 | } |
| 338 | } |
| 339 | |
amadan