| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "os" |
| 6 | "path/filepath" |
| 7 | "reflect" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/hostclaim" |
| 12 | "github.com/carlosframework/platform/internal/provision" |
| 13 | ) |
| 14 | |
| 15 | // The box's side of `move resolve --retry` (moves spec §7.8 stuck, §7.10; |
| 16 | // Plan 4 Task 7). The console door writes provision.MoveRetry and nothing |
| 17 | // else (h.retry is the same transition through the same record writer); the |
| 18 | // box then resumes the source's recovery at the phase the stuck rollback |
| 19 | // recorded as StuckAt, with the stuck reason gone and the rollback's own |
| 20 | // reason kept. A verification runs again and is never bypassed (INV-4); a |
| 21 | // probe that ran out of attempts is called again with the whole budget; |
| 22 | // nothing earlier than StuckAt runs again. The fixture is Task 5's. |
| 23 | |
| 24 | // verifications counts how many times the source ran its baseline check. |
| 25 | func (h *recoverHarness) verifications() int { |
| 26 | n := 0 |
| 27 | for _, p := range h.rcHits { |
| 28 | if p == "verify" { |
| 29 | n++ |
| 30 | } |
| 31 | } |
| 32 | return n |
| 33 | } |
| 34 | |
| 35 | // retried asserts what MoveRetry leaves for the box: rolling back at the |
| 36 | // phase the stuck mark named, no stuck reason, no stuck-at, the cause kept. |
| 37 | func (h *recoverHarness) retried(t *testing.T, at, cause string) { |
| 38 | t.Helper() |
| 39 | m := h.read(t, p1Host).Move |
| 40 | if m.State != provision.MoveRollingBack || m.StuckReason != "" || m.Rollback == nil || m.Rollback.SourcePhase != at || m.Rollback.StuckAt != "" { |
| 41 | t.Fatalf("after the retry: %+v %+v, want rolling_back at %s", m, m.Rollback, at) |
| 42 | } |
| 43 | if m.Rollback.Reason != cause { |
| 44 | t.Fatalf("the retry changed the rollback's cause: %q, want %q", m.Rollback.Reason, cause) |
| 45 | } |
| 46 | } |
| 47 | |
| 48 | // TestRetryResumesTheReturnsVerification (J20, INV-4): a clean return whose |
| 49 | // files no longer match the source's stopped baseline is stuck at verified. |
| 50 | // A retry returns the record to verified, and the box verifies again, once |
| 51 | // per retry; unrepaired it is stuck again with the same sentence and nothing |
| 52 | // after the check has run; repaired it recovers. The retry never skips the |
| 53 | // check, and never runs the hand-over, the claim write or anything before |
| 54 | // verified a second time. |
| 55 | func TestRetryResumesTheReturnsVerification(t *testing.T) { |
| 56 | t.Parallel() |
| 57 | h := newRecoverHarness(t, false) |
| 58 | rb := h.released(t) |
| 59 | cause := h.read(t, p1Host).Move.Rollback.Reason |
| 60 | upload := filepath.Join(h.sdir, p1Host, "upload.txt") |
| 61 | if err := os.WriteFile(upload, []byte("changed on the source"), 0o644); err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | stuck := func(verified int) { |
| 65 | t.Helper() |
| 66 | m := h.read(t, p1Host).Move |
| 67 | if m.State != provision.MoveStuck || m.StuckReason != provision.StuckBaselineMismatch || m.Rollback.StuckAt != provision.RbSourceVerified { |
| 68 | t.Fatalf("not stuck at verified: %+v %+v", m, m.Rollback) |
| 69 | } |
| 70 | if n := h.verifications(); n != verified { |
| 71 | t.Fatalf("verified %d times, want %d (%v)", n, verified, h.rcHits) |
| 72 | } |
| 73 | if len(h.probes) != 0 { |
| 74 | t.Fatalf("probed past a mismatch: %+v", h.probes) |
| 75 | } |
| 76 | } |
| 77 | h.srcPass(t) |
| 78 | stuck(1) |
| 79 | handed := len(h.srcHanded) |
| 80 | if handed != 1 { |
| 81 | t.Fatalf("hand-overs before the retry: %v", h.srcHanded) |
| 82 | } |
| 83 | // Stuck is not driven on: a pass between the stuck mark and the retry |
| 84 | // verifies nothing. |
| 85 | h.srcPass(t) |
| 86 | stuck(1) |
| 87 | |
| 88 | // The retry alone verifies nothing: it only moves the record. |
| 89 | h.retry(t) |
| 90 | h.retried(t, provision.RbSourceVerified, cause) |
| 91 | if n := h.verifications(); n != 1 { |
| 92 | t.Fatalf("the retry verified by itself: %d", n) |
| 93 | } |
| 94 | h.srcPass(t) |
| 95 | stuck(2) |
| 96 | h.retry(t) |
| 97 | h.srcPass(t) |
| 98 | stuck(3) |
| 99 | |
| 100 | // The operator repairs the source; one more retry reaches opened. |
| 101 | if err := os.WriteFile(upload, []byte("hello"), 0o644); err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | h.retry(t) |
| 105 | h.retried(t, provision.RbSourceVerified, cause) |
| 106 | h.srcPass(t) |
| 107 | got := h.opened(t) |
| 108 | if got.ID != rb.ID || got.StuckAt != "" || h.read(t, p1Host).Move.StuckReason != "" { |
| 109 | t.Fatalf("opened: %+v", got) |
| 110 | } |
| 111 | if n := h.verifications(); n != 4 { |
| 112 | t.Fatalf("verified %d times, want 4 (one per attempt)", n) |
| 113 | } |
| 114 | if len(h.srcHanded) != handed { |
| 115 | t.Fatalf("the retry handed the row over again: %v", h.srcHanded) |
| 116 | } |
| 117 | if len(h.probes) != 1 { |
| 118 | t.Fatalf("probes: %d, want the one after the repair", len(h.probes)) |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | // TestRetryGivesTheProbeItsWholeBudgetAgain (J20): a recovery probe that ran |
| 123 | // out of attempts is stuck at probing. A retry resumes at probing, so the |
| 124 | // box calls the probe again with the full nine attempts (the counter starts |
| 125 | // again at 0) and runs nothing before it: no verification, no install, no |
| 126 | // claim step. The probe keeps its own count (Task 9 builds it on claim |
| 127 | // Plan 10's attempts); the box's journal holds none to reset. |
| 128 | func TestRetryGivesTheProbeItsWholeBudgetAgain(t *testing.T) { |
| 129 | t.Parallel() |
| 130 | h := newRecoverHarness(t, false) |
| 131 | h.released(t) |
| 132 | h.probe = func(recoveryProbeRequest) (recoveryProbeOutcome, error) { return recoveryProbeExhausted, nil } |
| 133 | h.srcPass(t) |
| 134 | cause := h.read(t, p1Host).Move.Rollback.Reason |
| 135 | stuck := func(probes int) { |
| 136 | t.Helper() |
| 137 | m := h.read(t, p1Host).Move |
| 138 | if m.State != provision.MoveStuck || m.StuckReason != provision.StuckRecoveryProbeAttempts || m.Rollback.StuckAt != provision.RbSourceProbing { |
| 139 | t.Fatalf("not stuck at probing: %+v %+v", m, m.Rollback) |
| 140 | } |
| 141 | if len(h.probes) != probes { |
| 142 | t.Fatalf("probe calls: %d, want %d", len(h.probes), probes) |
| 143 | } |
| 144 | } |
| 145 | stuck(1) |
| 146 | h.srcPass(t) |
| 147 | stuck(1) // stuck is not driven on |
| 148 | // The source's files change under the stuck probe, as a live app's would. |
| 149 | if err := os.WriteFile(filepath.Join(h.sdir, p1Host, "upload.txt"), []byte("written by the recovered app"), 0o644); err != nil { |
| 150 | t.Fatal(err) |
| 151 | } |
| 152 | |
| 153 | h.retry(t) |
| 154 | h.retried(t, provision.RbSourceProbing, cause) |
| 155 | h.srcPass(t) |
| 156 | stuck(2) |
| 157 | h.retry(t) |
| 158 | h.probe = nil |
| 159 | h.srcPass(t) |
| 160 | h.opened(t) |
| 161 | if len(h.probes) != 3 { |
| 162 | t.Fatalf("probe calls: %d, want 3", len(h.probes)) |
| 163 | } |
| 164 | for i, req := range h.probes { |
| 165 | if req.Attempts != 9 { |
| 166 | t.Fatalf("probe call %d carries a budget of %d, want the whole nine", i, req.Attempts) |
| 167 | } |
| 168 | if !req.Clean || req.Coverage != recoveryCoverageClean || req.SourceGeneration != "cur" { |
| 169 | t.Fatalf("probe call %d: %+v", i, req) |
| 170 | } |
| 171 | } |
| 172 | if n := h.verifications(); n != 1 { |
| 173 | t.Fatalf("verified %d times, want 1: a retry at probing never verifies again (INV-2)", n) |
| 174 | } |
| 175 | if len(h.srcHanded) != 1 { |
| 176 | t.Fatalf("hand-overs: %v", h.srcHanded) |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | // TestRetryResumesADirtyInstallAtLocal: a dirty return's install that found |
| 181 | // the source's files occupied is stuck at local (the three install reasons |
| 182 | // are legal at local and configured). A retry resumes the shared installer |
| 183 | // at local from its journal, not from the start: the pre-move park, run |
| 184 | // once, is not run again, and once the operator has cleared the files the |
| 185 | // recovery reaches opened. |
| 186 | func TestRetryResumesADirtyInstallAtLocal(t *testing.T) { |
| 187 | t.Parallel() |
| 188 | h := newRecoverHarness(t, true) |
| 189 | h.released(t) |
| 190 | sentinel := filepath.Join(h.sdir, p1Host, "sentinel.txt") |
| 191 | h.landFault = func(p string) error { |
| 192 | if p == "after-"+landStepRetired { |
| 193 | if err := os.MkdirAll(filepath.Dir(sentinel), 0o755); err != nil { |
| 194 | t.Fatal(err) |
| 195 | } |
| 196 | return os.WriteFile(sentinel, []byte("not the return's"), 0o644) |
| 197 | } |
| 198 | return nil |
| 199 | } |
| 200 | h.srcPass(t) |
| 201 | cause := h.read(t, p1Host).Move.Rollback.Reason |
| 202 | m := h.read(t, p1Host).Move |
| 203 | if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnFilesOccupied || m.Rollback.StuckAt != provision.RbSourceLocal { |
| 204 | t.Fatalf("not stuck at local: %+v %+v", m, m.Rollback) |
| 205 | } |
| 206 | if len(h.probes) != 0 { |
| 207 | t.Fatalf("probed past a refused install: %+v", h.probes) |
| 208 | } |
| 209 | // Stuck, retried with the sentinel still there: stuck again, the same |
| 210 | // sentence, the sentinel untouched. |
| 211 | h.retry(t) |
| 212 | h.retried(t, provision.RbSourceLocal, cause) |
| 213 | h.srcPass(t) |
| 214 | if m := h.read(t, p1Host).Move; m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnFilesOccupied || m.Rollback.StuckAt != provision.RbSourceLocal { |
| 215 | t.Fatalf("an unrepaired retry: %+v %+v", m, m.Rollback) |
| 216 | } |
| 217 | if body, err := os.ReadFile(sentinel); err != nil || string(body) != "not the return's" { |
| 218 | t.Fatalf("the sentinel: %q %v", body, err) |
| 219 | } |
| 220 | // Repaired: the operator removes what is not the return's. |
| 221 | if err := os.Remove(sentinel); err != nil { |
| 222 | t.Fatal(err) |
| 223 | } |
| 224 | h.landFault = nil |
| 225 | h.retry(t) |
| 226 | h.srcPass(t) |
| 227 | rb := h.opened(t) |
| 228 | if body, err := os.ReadFile(provision.DefaultDBPath(h.sdir, p1Host) + ".gen"); err != nil || strings.TrimSpace(string(body)) != rb.ID { |
| 229 | t.Fatalf("the installed generation marker: %q %v", body, err) |
| 230 | } |
| 231 | if n := h.verifications(); n != 0 { |
| 232 | t.Fatalf("a dirty return verified the baseline %d times; it installs instead", n) |
| 233 | } |
| 234 | if len(h.probes) != 1 || h.probes[0].Clean || h.probes[0].Coverage != recoveryCoverageSealed || h.probes[0].Attempts != recoveryProbeAttempts { |
| 235 | t.Fatalf("probes: %+v", h.probes) |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | // TestRetryLeavesTheClaimAndTheRowAlone: the retry is a record write. It |
| 240 | // does not touch the host claim or the registry row; the box's next pass |
| 241 | // does what the phase asks. |
| 242 | func TestRetryLeavesTheClaimAndTheRowAlone(t *testing.T) { |
| 243 | t.Parallel() |
| 244 | h := newRecoverHarness(t, false) |
| 245 | h.released(t) |
| 246 | if err := os.WriteFile(filepath.Join(h.sdir, p1Host, "upload.txt"), []byte("changed"), 0o644); err != nil { |
| 247 | t.Fatal(err) |
| 248 | } |
| 249 | h.srcPass(t) |
| 250 | claim, row := h.claim(t), h.srcRoute |
| 251 | h.retry(t) |
| 252 | after := h.claim(t) |
| 253 | if after.Op.Phase != hostclaim.PhaseClaimed || after.Op.Phase != claim.Op.Phase || after.Op.Exec != claim.Op.Exec { |
| 254 | t.Fatalf("the retry changed the claim: %+v -> %+v", claim.Op, after.Op) |
| 255 | } |
| 256 | if !reflect.DeepEqual(h.srcRoute, row) { |
| 257 | t.Fatalf("the retry changed the row: %+v -> %+v", row, h.srcRoute) |
| 258 | } |
| 259 | // A second retry, now that the move is not stuck, is the provision |
| 260 | // refusal the door speaks as its 409. |
| 261 | g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc} |
| 262 | if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveRetry(g)); err == nil || !strings.Contains(err.Error(), provision.RefusedNotStuck) { |
| 263 | t.Fatalf("a retry of a move that is not stuck: %v", err) |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | // countEvent counts event in events. |
| 268 | func countEvent(events []string, event string) int { |
| 269 | n := 0 |
| 270 | for _, e := range events { |
| 271 | if e == event { |
| 272 | n++ |
| 273 | } |
| 274 | } |
| 275 | return n |
| 276 | } |
| 277 | |
| 278 | // Final review I3: a retry's event comes from the box's own pass. After |
| 279 | // `move resolve --retry` (the claim-backed MoveRetryRecovery, which advances |
| 280 | // the epoch) the record is rolling_back or reverting, which movesPass sends |
| 281 | // to the source's recovery or revert before phase1MoveRound; the pass still |
| 282 | // logs retry-<epoch>, once, however many passes see it. |
| 283 | func TestRetryEventIsLoggedByThePass(t *testing.T) { |
| 284 | t.Parallel() |
| 285 | t.Run("a rollback's source recovery", func(t *testing.T) { |
| 286 | h := newRecoverHarness(t, false) |
| 287 | h.released(t) |
| 288 | if err := os.WriteFile(filepath.Join(h.sdir, p1Host, "upload.txt"), []byte("changed on the source"), 0o644); err != nil { |
| 289 | t.Fatal(err) |
| 290 | } |
| 291 | h.srcPass(t) |
| 292 | if m := h.read(t, p1Host).Move; m.State != provision.MoveStuck { |
| 293 | t.Fatalf("not stuck: %+v", m) |
| 294 | } |
| 295 | h.retry(t) |
| 296 | if m := h.read(t, p1Host).Move; m.State != provision.MoveRollingBack || m.RecoveryRetryEpoch != 1 { |
| 297 | t.Fatalf("after the retry: %+v", m) |
| 298 | } |
| 299 | h.srcPass(t) |
| 300 | h.srcPass(t) |
| 301 | if n := countEvent(h.sEvents, "retry"); n != 1 { |
| 302 | t.Fatalf("retry events %d, want one: %v", n, h.sEvents) |
| 303 | } |
| 304 | }) |
| 305 | |
| 306 | t.Run("a revert", func(t *testing.T) { |
| 307 | h := newRevertHarness(t) |
| 308 | h.fromStopped(t) |
| 309 | h.arm(t) |
| 310 | if err := os.WriteFile(filepath.Join(h.cfg.DataDir, p1Host, "upload.txt"), []byte("changed while stopped"), 0o644); err != nil { |
| 311 | t.Fatal(err) |
| 312 | } |
| 313 | h.pass(t) |
| 314 | if m := h.move(t); m.State != provision.MoveStuck { |
| 315 | t.Fatalf("not stuck: %+v", m) |
| 316 | } |
| 317 | g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc} |
| 318 | if _, err := provision.SetRecordMove(context.Background(), h.s, "bab", "app", p1Host, provision.MoveRetryRecovery(g)); err != nil { |
| 319 | t.Fatal(err) |
| 320 | } |
| 321 | if m := h.move(t); m.State != provision.MoveReverting || m.RecoveryRetryEpoch != 1 { |
| 322 | t.Fatalf("after the retry: %+v", m) |
| 323 | } |
| 324 | h.pass(t) |
| 325 | h.pass(t) |
| 326 | if n := countEvent(h.events, "retry"); n != 1 { |
| 327 | t.Fatalf("retry events %d, want one: %v", n, h.events) |
| 328 | } |
| 329 | }) |
| 330 | } |
| 331 | |
amadan