| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "log" |
| 5 | "reflect" |
| 6 | "strings" |
| 7 | "testing" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/hostclaim" |
| 10 | "github.com/carlosframework/platform/internal/moves" |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | ) |
| 14 | |
| 15 | // count is how many times s was logged (housekeeper_test.go's buffer). |
| 16 | func (l *lockedBuffer) count(s string) int { return strings.Count(l.String(), s) } |
| 17 | |
| 18 | // lostLog sends the log to a buffer for the rest of the test; a pass may |
| 19 | // log from goroutines, so the buffer is housekeeper_test.go's locked one. |
| 20 | func lostLog(t *testing.T) *lockedBuffer { |
| 21 | t.Helper() |
| 22 | buf := &lockedBuffer{} |
| 23 | out, flags := log.Writer(), log.Flags() |
| 24 | log.SetOutput(buf) |
| 25 | t.Cleanup(func() { log.SetOutput(out); log.SetFlags(flags) }) |
| 26 | return buf |
| 27 | } |
| 28 | |
| 29 | // rcLose lists box in the host claim's Lost, as a declaration leaves it. |
| 30 | func rcLose(t *testing.T, h *phase1Harness, box string) { |
| 31 | t.Helper() |
| 32 | rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { |
| 33 | c.Lost = append(c.Lost, box) |
| 34 | if c.LostRevisions == nil { |
| 35 | c.LostRevisions = map[string]uint64{} |
| 36 | } |
| 37 | c.LostRevisions[box] = 1 |
| 38 | }) |
| 39 | } |
| 40 | |
| 41 | // Q9's gate (Plan 4 Task 10, witness wLostQuarantine; delta M15): a box |
| 42 | // the host's claim lists in Lost runs no move step for the host, whatever |
| 43 | // its role and the record's state, and logs one standing notice. The |
| 44 | // quarantine's local retirement is the claim's (readmission requires its |
| 45 | // receipt); moves adds only this gate. |
| 46 | func TestReturningLostBoxRefusesEveryMoveStep(t *testing.T) { |
| 47 | type snapshot struct { |
| 48 | rec provision.Record |
| 49 | claim hostclaim.Claim |
| 50 | } |
| 51 | take := func(t *testing.T, h *phase1Harness) snapshot { |
| 52 | t.Helper() |
| 53 | return snapshot{rec: h.read(t, p1Host), claim: *h.claim(t)} |
| 54 | } |
| 55 | t.Run("the source before the flip", func(t *testing.T) { |
| 56 | buf := lostLog(t) |
| 57 | h := newPhase1Harness(t) |
| 58 | h.start(t, "auto") |
| 59 | rcLose(t, h, "ie-box") |
| 60 | before := take(t, h) |
| 61 | h.pass(t) |
| 62 | h.pass(t) |
| 63 | if after := take(t, h); !reflect.DeepEqual(before, after) { |
| 64 | t.Fatalf("a lost source moved the host:\n%+v\n%+v", before.rec.Move, after.rec.Move) |
| 65 | } |
| 66 | if r, err := readSourceReport(t.Context(), h.s, h.read(t, p1Host)); err != nil || r != nil { |
| 67 | t.Fatalf("a lost source reported: %+v %v", r, err) |
| 68 | } |
| 69 | if n := buf.count("is lost on this host's claim"); n != 1 { |
| 70 | t.Fatalf("notices %d:\n%s", n, buf.String()) |
| 71 | } |
| 72 | // The gate is this box's: the same record moves on once the claim |
| 73 | // no longer lists it. |
| 74 | rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { c.Lost, c.LostRevisions = nil, nil }) |
| 75 | h.pass(t) |
| 76 | if r, err := readSourceReport(t.Context(), h.s, h.read(t, p1Host)); err != nil || r == nil { |
| 77 | t.Fatalf("the readmitted source did not report: %v", err) |
| 78 | } |
| 79 | }) |
| 80 | t.Run("the target before the flip", func(t *testing.T) { |
| 81 | lostLog(t) |
| 82 | h := newPhase1Harness(t) |
| 83 | h.start(t, "auto") |
| 84 | h.pass(t) |
| 85 | rcLose(t, h, "uk-box") |
| 86 | before := take(t, h) |
| 87 | h.targetPass(t, h.targetCfg()) |
| 88 | if after := take(t, h); !reflect.DeepEqual(before, after) { |
| 89 | t.Fatalf("a lost target moved the host:\n%+v\n%+v", before.rec.Move, after.rec.Move) |
| 90 | } |
| 91 | if r, err := readTargetReport(t.Context(), h.s, h.read(t, p1Host)); err != nil || r != nil { |
| 92 | t.Fatalf("a lost target reported: %+v %v", r, err) |
| 93 | } |
| 94 | }) |
| 95 | t.Run("the source's recovery", func(t *testing.T) { |
| 96 | buf := lostLog(t) |
| 97 | h := newRecoverHarness(t, false) |
| 98 | h.released(t) |
| 99 | rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { |
| 100 | c.Lost, c.LostRevisions = append(c.Lost, "ie-box"), map[string]uint64{"ie-box": 1} |
| 101 | }) |
| 102 | rec, claim := h.read(t, p1Host), *h.claim(t) |
| 103 | h.srcPass(t) |
| 104 | if !reflect.DeepEqual(rec, h.read(t, p1Host)) || !reflect.DeepEqual(claim, *h.claim(t)) || len(h.probes) != 0 || h.srcRoute.HoldOp != p1ID { |
| 105 | t.Fatalf("a lost source recovered: %+v", h.read(t, p1Host).Move.Rollback) |
| 106 | } |
| 107 | if n := buf.count("is lost on this host's claim"); n != 1 { |
| 108 | t.Fatalf("notices %d", n) |
| 109 | } |
| 110 | }) |
| 111 | } |
| 112 | |
| 113 | // rcCorruptClaim makes the host claim unreadable: its object no longer |
| 114 | // decodes. |
| 115 | func rcCorruptClaim(t *testing.T, s release.Store) { |
| 116 | t.Helper() |
| 117 | key, err := hostclaim.Key(p1Host) |
| 118 | if err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | _, etag, err := s.Get(t.Context(), key) |
| 122 | if err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | if _, err := s.Put(t.Context(), key, []byte("{not a claim"), etag); err != nil { |
| 126 | t.Fatal(err) |
| 127 | } |
| 128 | } |
| 129 | |
| 130 | // Fix round 1, item 3: the gate covers the whole pass. A host the gate |
| 131 | // blocks, confirmed lost or with a claim that does not read, gets no target |
| 132 | // report and no journal sweep: nothing is reported, committed or retired. |
| 133 | func TestLostBoxGateCoversTheWholePass(t *testing.T) { |
| 134 | t.Run("an unreadable claim blocks the target's report", func(t *testing.T) { |
| 135 | buf := lostLog(t) |
| 136 | h := newPhase1Harness(t) |
| 137 | h.start(t, "auto") |
| 138 | h.pass(t) |
| 139 | rcCorruptClaim(t, h.s) |
| 140 | h.targetPass(t, h.targetCfg()) |
| 141 | if r, err := readTargetReport(t.Context(), h.s, h.read(t, p1Host)); err != nil || r != nil { |
| 142 | t.Fatalf("a target with an unreadable claim reported: %+v %v", r, err) |
| 143 | } |
| 144 | if buf.count("does not read") == 0 { |
| 145 | t.Fatalf("no notice:\n%s", buf.String()) |
| 146 | } |
| 147 | }) |
| 148 | for name, block := range map[string]func(t *testing.T, h *revertHarness){ |
| 149 | "a confirmed lost host": func(t *testing.T, h *revertHarness) { |
| 150 | rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { |
| 151 | c.Lost, c.LostRevisions = append(c.Lost, "ie-box"), map[string]uint64{"ie-box": 1} |
| 152 | }) |
| 153 | }, |
| 154 | "an unreadable claim": func(t *testing.T, h *revertHarness) { rcCorruptClaim(t, h.s) }, |
| 155 | } { |
| 156 | t.Run(name+" blocks the journal sweep", func(t *testing.T) { |
| 157 | lostLog(t) |
| 158 | h := newRevertHarness(t) |
| 159 | h.fromStopped(t) |
| 160 | h.arm(t) |
| 161 | h.pass(t) |
| 162 | h.reopened(t) |
| 163 | h.rvAt = "after-close" |
| 164 | o := h.laneChore(t) |
| 165 | h.tPass(t) |
| 166 | h.pastHorizon(t, o) |
| 167 | h.tPass(t) |
| 168 | h.pass(t) |
| 169 | if !h.rvDone || h.read(t, p1Host).Move != nil || !moveJournalPresent(h.cfg.DataDir, p1Host, p1ID) { |
| 170 | t.Fatalf("the setup did not leave the closed revert's journal: %v", h.rvHits) |
| 171 | } |
| 172 | journal, err := moves.ReadJournal(h.cfg.DataDir, p1Host, p1ID) |
| 173 | if err != nil { |
| 174 | t.Fatal(err) |
| 175 | } |
| 176 | block(t, h) |
| 177 | h.freshWorker() |
| 178 | h.pass(t) |
| 179 | if !moveJournalPresent(h.cfg.DataDir, p1Host, p1ID) { |
| 180 | t.Fatal("the sweep retired a blocked host's journal") |
| 181 | } |
| 182 | if after, err := moves.ReadJournal(h.cfg.DataDir, p1Host, p1ID); err != nil || !reflect.DeepEqual(after.Steps, journal.Steps) { |
| 183 | t.Fatalf("the sweep committed a blocked host's journal: %v", err) |
| 184 | } |
| 185 | }) |
| 186 | } |
| 187 | } |
| 188 | |
| 189 | // Fix round 1, item 4: the gate names this box's region through the shared |
| 190 | // placement rule (park.PlacementMatches under the verified home binding), |
| 191 | // never raw string equality: on the home box (placement code "") a record |
| 192 | // that spells the home by its roster name names this box, and is gated; |
| 193 | // without the verified binding it names another region. |
| 194 | func TestLostBoxGateReadsHomeSpellingsUnderTheBinding(t *testing.T) { |
| 195 | s := release.NewDirStore(t.TempDir()) |
| 196 | inc, err := hostclaim.Incarnation("bab", "app", p1Host, p1Created) |
| 197 | if err != nil { |
| 198 | t.Fatal(err) |
| 199 | } |
| 200 | if _, err := hostclaim.Update(t.Context(), s, hostclaim.Observed{Host: p1Host, Absent: true}, func(c *hostclaim.Claim) error { |
| 201 | c.Holder = &hostclaim.Holder{Family: "instance", Key: provision.RecordKey("bab", "app", p1Host), Account: "bab", App: "app", Incarnation: inc, CreatedAt: p1Created, Location: "instances/" + p1Host, Generation: "cur", Stamp: 1} |
| 202 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "home-box"} |
| 203 | c.Lost, c.LostRevisions = []string{"home-box"}, map[string]uint64{"home-box": 1} |
| 204 | return nil |
| 205 | }); err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | for _, spelling := range []provision.Record{ |
| 209 | {Host: p1Host, Region: "ie", Move: &provision.MoveState{ID: p1ID, From: "ie", To: "uk"}}, |
| 210 | {Host: p1Host, Region: "uk", Move: &provision.MoveState{ID: p1ID, From: "uk", To: "ie"}}, |
| 211 | {Host: p1Host, Region: "", Move: &provision.MoveState{ID: p1ID, From: "", To: "uk"}}, |
| 212 | } { |
| 213 | bound := &movesWorker{cfg: moveCfg{Box: "home-box", Region: "", ParkHomeRegion: "ie"}} |
| 214 | if !movesNamesBox(bound.cfg, spelling) || !bound.lostHere(t.Context(), s, spelling) { |
| 215 | t.Fatalf("the home box under the binding does not gate %+v", spelling.Move) |
| 216 | } |
| 217 | } |
| 218 | unbound := moveCfg{Box: "home-box", Region: ""} |
| 219 | if movesNamesBox(unbound, provision.Record{Host: p1Host, Region: "ie", Move: &provision.MoveState{ID: p1ID, From: "ie", To: "uk"}}) { |
| 220 | t.Fatal("a roster name matched the home without the verified binding") |
| 221 | } |
| 222 | named := moveCfg{Box: "de-box", Region: "de", ParkHomeRegion: "ie"} |
| 223 | if movesNamesBox(named, provision.Record{Host: p1Host, Region: "", Move: &provision.MoveState{ID: p1ID, From: "", To: "uk"}}) { |
| 224 | t.Fatal("a named box matched the home") |
| 225 | } |
| 226 | } |
| 227 | |
amadan