| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "reflect" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/hostclaim" |
| 12 | "github.com/carlosframework/platform/internal/moves" |
| 13 | "github.com/carlosframework/platform/internal/provision" |
| 14 | ) |
| 15 | |
| 16 | // probedLanding is a landing driven from the flip through every phase with |
| 17 | // the real transitions, to probed. |
| 18 | func probedLanding(t *testing.T) (*landingHarness, *probeLog) { |
| 19 | t.Helper() |
| 20 | h, p := installedLanding(t, 2) |
| 21 | h.land(t) |
| 22 | h.atPhase(t, provision.LandProbed) |
| 23 | h.holdProbed = false |
| 24 | return h, p |
| 25 | } |
| 26 | |
| 27 | // committedOpened asserts the end state of the commit and the open: the |
| 28 | // record committed with opened_at, the claim committed with the admitted |
| 29 | // writer kept and the commit's parks queued, the row ready, one probe. |
| 30 | func committedOpened(t *testing.T, h *landingHarness, p *probeLog) { |
| 31 | t.Helper() |
| 32 | m := h.read(t, p1Host).Move |
| 33 | if m.State != provision.MoveCommitted || m.CommittedAt == nil || m.OpenedAt == nil { |
| 34 | t.Fatalf("record: %+v", m) |
| 35 | } |
| 36 | c := h.claim(t) |
| 37 | if c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseCommitted { |
| 38 | t.Fatalf("claim: %+v", c.Op) |
| 39 | } |
| 40 | if c.Holder.Session != p.last.Session || c.Holder.Stamp != p.last.Stamp || c.Holder.Generation != p.last.Generation { |
| 41 | t.Fatalf("the commit changed the admitted writer %+v: %+v", p.last, c.Holder) |
| 42 | } |
| 43 | kinds := map[string]int{} |
| 44 | m2 := h.read(t, p1Host).Move |
| 45 | for _, o := range c.Obligations { |
| 46 | kinds[o.Kind+"/"+o.Region]++ |
| 47 | if o.Kind == "park-replica" && o.Region == "ie" && (o.Park == nil || o.Park.Location != c.Holder.Location || !o.Park.EvidenceAt.Equal(m2.FlippedAt.UTC())) { |
| 48 | t.Fatalf("the source's park is not scoped from the flip: %+v", o.Park) |
| 49 | } |
| 50 | if o.Kind == "park-replica" && o.Region == "uk" && (o.Park == nil || o.Park.Location != "moves/"+p1Host+"/"+p1ID) { |
| 51 | t.Fatalf("the staging park: %+v", o.Park) |
| 52 | } |
| 53 | } |
| 54 | // The source region's location (scoped from the flip, H7) and the |
| 55 | // target region's staging lane (scoped to the seal's bound). |
| 56 | if kinds["park-replica/ie"] != 1 || kinds["park-replica/uk"] != 1 { |
| 57 | t.Fatalf("the commit's parks: %+v", c.Obligations) |
| 58 | } |
| 59 | if h.route.Status != "ready" || h.route.HoldOp != "" { |
| 60 | t.Fatalf("the row is not open: %+v", h.route) |
| 61 | } |
| 62 | if len(p.configs) != 1 { |
| 63 | t.Fatalf("probed %d times", len(p.configs)) |
| 64 | } |
| 65 | } |
| 66 | |
| 67 | // TestCommitThenOpenInOrder (brief; witness wCommitBeforeOpen): from the |
| 68 | // flip through every landing phase with the real transitions, the commit is |
| 69 | // the record's CAS, then the claim's; the probe's live entry is marked open, |
| 70 | // and only then is the hold released: the row is ready only after the record |
| 71 | // and the claim both say committed. |
| 72 | func TestCommitThenOpenInOrder(t *testing.T) { |
| 73 | t.Parallel() |
| 74 | h, p := probedLanding(t) |
| 75 | h.land(t) |
| 76 | committedOpened(t, h, p) |
| 77 | want := []string{"mark:committed/committed/landing", "release:committed/committed/landing", "confirm:committed/committed/ready"} |
| 78 | if !reflect.DeepEqual(h.openLog, want) { |
| 79 | t.Fatalf("the open: %v, want %v", h.openLog, want) |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | // TestOpenAfterCrashDoesNotReprobe (brief, Review Focus 3): a worker |
| 84 | // restarted on a record at committed with the claim still landing/probed |
| 85 | // writes the claim half, opens, and never calls the probe. |
| 86 | func TestOpenAfterCrashDoesNotReprobe(t *testing.T) { |
| 87 | t.Parallel() |
| 88 | h, p := probedLanding(t) |
| 89 | h.landAt = "after-record-committed" |
| 90 | h.crashLand(t) |
| 91 | if m := h.read(t, p1Host).Move; m.State != provision.MoveCommitted || h.claim(t).Op.Phase != hostclaim.PhaseLandingProbed || h.route.Status != "landing" { |
| 92 | t.Fatalf("the crash point: %+v claim %s row %s", m, h.claim(t).Op.Phase, h.route.Status) |
| 93 | } |
| 94 | h.freshTarget() |
| 95 | h.land(t) |
| 96 | committedOpened(t, h, p) |
| 97 | } |
| 98 | |
| 99 | // The open repeats while the record is committed: a pass that finds the row |
| 100 | // already open changes nothing, opened_at included. |
| 101 | func TestCommittedOpenRepeatsConditionally(t *testing.T) { |
| 102 | t.Parallel() |
| 103 | h, p := probedLanding(t) |
| 104 | h.land(t) |
| 105 | committedOpened(t, h, p) |
| 106 | opened := *h.read(t, p1Host).Move.OpenedAt |
| 107 | h.clock.Advance(time.Minute) |
| 108 | h.land(t) |
| 109 | committedOpened(t, h, p) |
| 110 | if m := h.read(t, p1Host).Move; !m.OpenedAt.Equal(opened) { |
| 111 | t.Fatalf("opened_at moved: %v, was %v", m.OpenedAt, opened) |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | // TestLateCommitRollsBack (brief, Review Focus 2; H8): a commit that would |
| 116 | // come after the landing's deadline is refused by the record's CAS and |
| 117 | // becomes the commit-deadline rollback, at the one conversion site; the row |
| 118 | // never opens. |
| 119 | func TestLateCommitRollsBack(t *testing.T) { |
| 120 | t.Parallel() |
| 121 | h, _ := probedLanding(t) |
| 122 | h.landFault = func(point string) error { |
| 123 | if point == "before-commit" { |
| 124 | h.clock.Advance(cutLandingWindow + time.Minute) |
| 125 | } |
| 126 | return nil |
| 127 | } |
| 128 | h.land(t) |
| 129 | if m := h.read(t, p1Host).Move; m.State != provision.MoveLanding || m.Reason != provision.RollbackCommitDeadline { |
| 130 | t.Fatalf("a late commit: %+v", m) |
| 131 | } |
| 132 | h.landFault = nil |
| 133 | h.land(t) |
| 134 | rolledBack(t, h, provision.RollbackCommitDeadline, provision.LandProbed) |
| 135 | if h.route.Status == "ready" || len(h.openLog) != 0 { |
| 136 | t.Fatalf("a rolled-back landing opened: %+v %v", h.route, h.openLog) |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | // m-2: a claim lagging at probing behind a probed record is walked to probed |
| 141 | // before the commit, on the admitted session, so Commit (probed only) holds. |
| 142 | func TestCommitReassertsALaggingProbedClaim(t *testing.T) { |
| 143 | t.Parallel() |
| 144 | h, p := probedLanding(t) |
| 145 | rewriteClaimOp(t, h.s, func(op *hostclaim.Op) { op.Phase = hostclaim.PhaseLandingProbing }) |
| 146 | h.land(t) |
| 147 | committedOpened(t, h, p) |
| 148 | } |
| 149 | |
| 150 | // Unarmed: with no release of the hold the committed arm waits; the commit |
| 151 | // itself is the record's and is made. |
| 152 | func TestCommittedOpenWaitsUnarmed(t *testing.T) { |
| 153 | t.Parallel() |
| 154 | h, _ := probedLanding(t) |
| 155 | h.noRelease = true |
| 156 | h.land(t) |
| 157 | if m := h.read(t, p1Host).Move; m.State != provision.MoveCommitted || m.OpenedAt != nil || h.route.Status != "landing" { |
| 158 | t.Fatalf("an unarmed open: %+v %+v", m, h.route) |
| 159 | } |
| 160 | _ = context.Background |
| 161 | } |
| 162 | |
| 163 | // The commit's scopes are Plan 5 Task 2's helpers' (H7): the source's |
| 164 | // location from the flip, the staging lane to the seal's bound. |
| 165 | func TestCommitScopesAreThePlan5Helpers(t *testing.T) { |
| 166 | t.Parallel() |
| 167 | h, _ := probedLanding(t) |
| 168 | h.land(t) |
| 169 | rec := h.read(t, p1Host) |
| 170 | c := h.claim(t) |
| 171 | seal, err := journalledSeal(c) |
| 172 | if err != nil { |
| 173 | t.Fatal(err) |
| 174 | } |
| 175 | wantSource := moves.SourceLocationScope(*c.Holder, *rec.Move.FlippedAt) |
| 176 | wantStaging := moves.StagingScope(p1Host, p1ID, c.Holder.Incarnation, seal.Bound()) |
| 177 | var source, staging bool |
| 178 | for _, o := range c.Obligations { |
| 179 | if o.Kind != "park-replica" || o.Park == nil { |
| 180 | continue |
| 181 | } |
| 182 | switch *o.Park { |
| 183 | case wantSource: |
| 184 | source = true |
| 185 | case wantStaging: |
| 186 | staging = true |
| 187 | } |
| 188 | } |
| 189 | if !source || !staging { |
| 190 | t.Fatalf("the commit's parks are not the helpers' %+v and %+v: %+v", wantSource, wantStaging, c.Obligations) |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | // Task 7 fix round 1, finding 3: the open is proved, never assumed. An |
| 195 | // unheld row that is not ready is no replay of the open, and a release that |
| 196 | // matched nothing is refused: opened_at is not stamped in either case. |
| 197 | func TestCommittedOpenIsProved(t *testing.T) { |
| 198 | t.Parallel() |
| 199 | h, _ := probedLanding(t) |
| 200 | h.route.HoldOp = "" |
| 201 | h.land(t) |
| 202 | if m := h.read(t, p1Host).Move; m.State != provision.MoveCommitted || m.OpenedAt != nil { |
| 203 | t.Fatalf("an unheld row that is not ready was taken as open: %+v", m) |
| 204 | } |
| 205 | |
| 206 | h2, _ := probedLanding(t) |
| 207 | h2.releaseNoop = true |
| 208 | h2.land(t) |
| 209 | if m := h2.read(t, p1Host).Move; m.State != provision.MoveCommitted || m.OpenedAt != nil || h2.route.Status == "ready" { |
| 210 | t.Fatalf("a release that matched nothing was taken as the open: %+v %+v", m, h2.route) |
| 211 | } |
| 212 | // The release happened and the row is ready: the next pass stamps it. |
| 213 | h2.releaseNoop = false |
| 214 | h2.land(t) |
| 215 | if m := h2.read(t, p1Host).Move; m.OpenedAt == nil || h2.route.Status != "ready" { |
| 216 | t.Fatalf("the open after the refusal: %+v %+v", m, h2.route) |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | // Final review, ruling 3: the probe entry's idle protection ends only on a |
| 221 | // proved release. A release that fails confirms nothing and stamps |
| 222 | // nothing; the retried open that succeeds confirms it. |
| 223 | func TestCommittedOpenConfirmsOnlyAProvedRelease(t *testing.T) { |
| 224 | t.Parallel() |
| 225 | h, _ := probedLanding(t) |
| 226 | h.releaseErr = errors.New("the registry is busy") |
| 227 | h.land(t) |
| 228 | for _, step := range h.openLog { |
| 229 | if strings.HasPrefix(step, "confirm:") { |
| 230 | t.Fatalf("a failed release was confirmed: %v", h.openLog) |
| 231 | } |
| 232 | } |
| 233 | if m := h.read(t, p1Host).Move; m.OpenedAt != nil { |
| 234 | t.Fatalf("a failed release was stamped: %+v", m) |
| 235 | } |
| 236 | h.releaseErr = nil |
| 237 | h.land(t) |
| 238 | if last := h.openLog[len(h.openLog)-1]; !strings.HasPrefix(last, "confirm:") || h.read(t, p1Host).Move.OpenedAt == nil { |
| 239 | t.Fatalf("the retried open: %v", h.openLog) |
| 240 | } |
| 241 | } |
| 242 | |
amadan