| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "github.com/carlosframework/platform/internal/rootstate" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "strings" |
| 12 | "testing" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/hostactors" |
| 15 | "github.com/carlosframework/platform/internal/hostfloor" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | "github.com/carlosframework/platform/internal/rootparticipation" |
| 18 | ) |
| 19 | |
| 20 | // carlos-system-recover runs the root generation census to settle the |
| 21 | // participation transaction an unfinished apply left open. The edge's guard |
| 22 | // drop-in Requires= the recover unit, so a recover that fails keeps the |
| 23 | // edge down. With no journal open the files are whole, so a census refusal |
| 24 | // defers the settlement instead: recover exits 0 with the transaction still |
| 25 | // open. With a journal open it fails as before. |
| 26 | |
| 27 | // harnessRefusal is the kernel census's refusal on the harness box, as |
| 28 | // productionGenerationKernelCensus returns it and the launch verifier |
| 29 | // passes it up unchanged. |
| 30 | func harnessRefusal() error { |
| 31 | return &generationCensusRefusal{err: fmt.Errorf("/proc/1535/task/1535: service: %w", os.ErrPermission)} |
| 32 | } |
| 33 | |
| 34 | // refuseCensus makes root's launch census refuse from its pass-th call on |
| 35 | // (1 is every call), and stubs the pending controller recovery so the |
| 36 | // refusal is the census's alone. |
| 37 | func refuseCensusFrom(t *testing.T, f *applyFixture, pass int) *int { |
| 38 | t.Helper() |
| 39 | calls := 0 |
| 40 | f.a.participation.VerifyLaunches = func(context.Context, uint64) error { |
| 41 | calls++ |
| 42 | if calls < pass { |
| 43 | return nil |
| 44 | } |
| 45 | return harnessRefusal() |
| 46 | } |
| 47 | old := recoverPendingControllers |
| 48 | t.Cleanup(func() { recoverPendingControllers = old }) |
| 49 | recoverPendingControllers = func(context.Context, string, string) error { return nil } |
| 50 | return &calls |
| 51 | } |
| 52 | |
| 53 | func refuseCensus(t *testing.T, f *applyFixture) *int { return refuseCensusFrom(t, f, 1) } |
| 54 | |
| 55 | func logLines(f *applyFixture) *[]string { |
| 56 | var lines []string |
| 57 | logf := f.a.ops.logf |
| 58 | f.a.ops.logf = func(format string, args ...any) { |
| 59 | lines = append(lines, fmt.Sprintf(format, args...)) |
| 60 | logf(format, args...) |
| 61 | } |
| 62 | return &lines |
| 63 | } |
| 64 | |
| 65 | func TestRecoverDefersACensusRefusalWithNoJournalOpen(t *testing.T) { |
| 66 | for _, marker := range []bool{true, false} { |
| 67 | t.Run(map[bool]string{true: "marker", false: "no-marker"}[marker], func(t *testing.T) { |
| 68 | f, id := guardRecoveryFixture(t) |
| 69 | a := f.a |
| 70 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 71 | t.Fatal(err) |
| 72 | } |
| 73 | begun := filepath.Join(a.floor.Dir, "update-begun.json") |
| 74 | if !marker { |
| 75 | if err := os.Remove(begun); err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | } |
| 79 | calls := refuseCensus(t, f) |
| 80 | lines := logLines(f) |
| 81 | if err := a.Run(t.Context(), true); err != nil { |
| 82 | t.Fatalf("recover with no journal open = %v, want a deferral that exits 0", err) |
| 83 | } |
| 84 | // Once before root's pending-controller recovery and once after, |
| 85 | // and not again at the end of the run. |
| 86 | if *calls != 2 { |
| 87 | t.Fatalf("recover ran the census %d times, want 2", *calls) |
| 88 | } |
| 89 | // The transaction is still open: no completion, the reservation |
| 90 | // kept, root's update record kept, so acquisitions stay refused. |
| 91 | if _, err := os.Stat(filepath.Join(a.floor.Dir, "completed.json")); !os.IsNotExist(err) { |
| 92 | t.Fatalf("a refused census was settled: %v", err) |
| 93 | } |
| 94 | if _, err := os.Stat(filepath.Join(a.floor.Dir, "activation.json")); err != nil { |
| 95 | t.Fatalf("reservation discarded: %v", err) |
| 96 | } |
| 97 | if _, err := os.Stat(begun); marker && err != nil { |
| 98 | t.Fatalf("marker discarded: %v", err) |
| 99 | } |
| 100 | if got, err := a.participation.UpdateID(); err != nil || got != id { |
| 101 | t.Fatalf("update record = %q, %v; want %s still open", got, err, id) |
| 102 | } |
| 103 | if err := a.participation.Acquire(t.Context(), func() error { return nil }); err == nil { |
| 104 | t.Fatal("a claim acquisition started while the transaction was open") |
| 105 | } |
| 106 | if len(f.calls) != 0 { |
| 107 | t.Fatal("recovery executed a binary or touched a unit", f.calls) |
| 108 | } |
| 109 | if !strings.Contains(strings.Join(*lines, "\n"), "service: permission denied") { |
| 110 | t.Fatalf("the deferral did not log the census's refusal: %q", *lines) |
| 111 | } |
| 112 | }) |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | func TestRecoverFailsACensusRefusalWithAJournalOpen(t *testing.T) { |
| 117 | f, id := guardRecoveryFixture(t) |
| 118 | a := f.a |
| 119 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | if err := a.journal(systemJournal{Phase: systemPhaseInstalled, Target: "new-build", PrevBuild: "old-build", StartedAt: f.now}); err != nil { |
| 123 | t.Fatal(err) |
| 124 | } |
| 125 | refuseCensus(t, f) |
| 126 | err := a.Run(t.Context(), true) |
| 127 | if err == nil || errors.Is(err, errSettlementDeferred) || !strings.Contains(err.Error(), "service: permission denied") { |
| 128 | t.Fatalf("recover with a journal open = %v, want the census's refusal", err) |
| 129 | } |
| 130 | if _, err := os.Stat(a.file("journal.json")); err != nil { |
| 131 | t.Fatalf("journal discarded: %v", err) |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | func TestRecoverDefersACensusRefusalWithoutAnArmedFloor(t *testing.T) { |
| 136 | // checkUnarmedParticipationRecovery: an apply began root's update |
| 137 | // record and stopped before writing a journal. |
| 138 | f := newApplyFixture(t, "stable", "old") |
| 139 | if f.a.guarded() { |
| 140 | t.Fatal("fixture has an armed floor") |
| 141 | } |
| 142 | if err := f.a.beginExecutableChange(t.Context(), "new", f.now); err != nil { |
| 143 | t.Fatal(err) |
| 144 | } |
| 145 | calls := refuseCensus(t, f) |
| 146 | if err := f.a.Run(t.Context(), true); err != nil { |
| 147 | t.Fatalf("recover with no journal open = %v, want a deferral that exits 0", err) |
| 148 | } |
| 149 | if *calls != 2 { |
| 150 | t.Fatalf("recover ran the census %d times, want 2 (not again to settle the marker)", *calls) |
| 151 | } |
| 152 | if _, err := f.a.participation.UpdateID(); err != nil { |
| 153 | t.Fatalf("recover settled a refused census: %v", err) |
| 154 | } |
| 155 | // With a journal open it still fails. |
| 156 | if err := f.a.journal(systemJournal{Phase: systemPhaseInstalling, Target: "new"}); err != nil { |
| 157 | t.Fatal(err) |
| 158 | } |
| 159 | if err := f.a.Run(t.Context(), true); err == nil { |
| 160 | t.Fatal("recover with a journal open passed a refused census") |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | func TestApplyStillFailsACensusRefusalWithNoJournalOpen(t *testing.T) { |
| 165 | // Only --recover defers: the apply's own start-up check returns the |
| 166 | // refusal as on main. |
| 167 | f, id := guardRecoveryFixture(t) |
| 168 | if err := f.a.floor.Begin(t.Context(), id, f.a.dataDir); err != nil { |
| 169 | t.Fatal(err) |
| 170 | } |
| 171 | refuseCensus(t, f) |
| 172 | if err := f.a.Run(t.Context(), false); err == nil || !strings.Contains(err.Error(), "service: permission denied") { |
| 173 | t.Fatalf("apply = %v, want the census's refusal", err) |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | func TestRuling6NoCensusRecoverIsUnchanged(t *testing.T) { |
| 178 | // A box without root participation (no census.json) never runs the |
| 179 | // generation census, and a refused settlement fails recover exactly as |
| 180 | // on main: the deferral is for census boxes only. |
| 181 | f, id := guardRecoveryFixture(t) |
| 182 | a := f.a |
| 183 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 184 | t.Fatal(err) |
| 185 | } |
| 186 | if err := a.participation.State.Remove("census.json"); err != nil { |
| 187 | t.Fatal(err) |
| 188 | } |
| 189 | if required, err := a.participationDisk().ParticipationRequired(t.Context()); err != nil || required { |
| 190 | t.Fatalf("fixture still requires participation: %t %v", required, err) |
| 191 | } |
| 192 | a.participation.VerifyLaunches = func(context.Context, uint64) error { |
| 193 | t.Fatal("a no-census box ran the generation census") |
| 194 | return nil |
| 195 | } |
| 196 | if err := os.WriteFile(filepath.Join(a.dataDir, hostexecDir, "host.json"), []byte(`{"state":"pending"}`), 0600); err != nil { |
| 197 | t.Fatal(err) |
| 198 | } |
| 199 | err := a.Run(t.Context(), true) |
| 200 | if err == nil || errors.Is(err, errSettlementDeferred) { |
| 201 | t.Fatalf("no-census recover = %v, want main's refusal", err) |
| 202 | } |
| 203 | if _, err := os.Stat(filepath.Join(a.floor.Dir, "completed.json")); !os.IsNotExist(err) { |
| 204 | t.Fatalf("completion written: %v", err) |
| 205 | } |
| 206 | // No census refusal can reach a no-census box's settlement, and one |
| 207 | // that did would not defer either. |
| 208 | if err := a.settleWithoutJournal(t.Context(), "recover-system", harnessRefusal()); errors.Is(err, errSettlementDeferred) { |
| 209 | t.Fatalf("a no-census box deferred: %v", err) |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | // openTransaction checks the guarded transaction is still open after a |
| 214 | // deferral: the reservation and root's update record kept, so claim |
| 215 | // acquisitions stay refused. marker says whether root's marker must be |
| 216 | // there too. |
| 217 | func openTransaction(t *testing.T, f *applyFixture, id string, marker bool) { |
| 218 | t.Helper() |
| 219 | a := f.a |
| 220 | if _, err := os.Stat(filepath.Join(a.floor.Dir, "activation.json")); err != nil { |
| 221 | t.Fatalf("reservation discarded: %v", err) |
| 222 | } |
| 223 | if _, err := os.Stat(filepath.Join(a.floor.Dir, "update-begun.json")); marker && err != nil { |
| 224 | t.Fatalf("marker discarded: %v", err) |
| 225 | } |
| 226 | if got, err := a.participation.UpdateID(); err != nil || got != id { |
| 227 | t.Fatalf("update record = %q, %v; want %s still open", got, err, id) |
| 228 | } |
| 229 | if err := a.participation.Acquire(t.Context(), func() error { return nil }); err == nil { |
| 230 | t.Fatal("a claim acquisition started while the transaction was open") |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | func TestRecoverDefersACensusRefusalOnlyAtTheEndOfParticipation(t *testing.T) { |
| 235 | // The pending-controller census passes; the one inside EndParticipation |
| 236 | // (EndUpdateWith -> idle) refuses. A finished transaction. |
| 237 | f, id := guardRecoveryFixture(t) |
| 238 | a := f.a |
| 239 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 240 | t.Fatal(err) |
| 241 | } |
| 242 | calls := refuseCensusFrom(t, f, 2) |
| 243 | if err := a.Run(t.Context(), true); err != nil { |
| 244 | t.Fatalf("recover = %v, want a deferral that exits 0", err) |
| 245 | } |
| 246 | if *calls != 2 { |
| 247 | t.Fatalf("census ran %d times, want 2 (controllers, then the end of participation)", *calls) |
| 248 | } |
| 249 | openTransaction(t, f, id, true) |
| 250 | // The next recover, with the census passing, settles it. |
| 251 | a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil } |
| 252 | if err := a.Run(t.Context(), true); err != nil { |
| 253 | t.Fatal(err) |
| 254 | } |
| 255 | if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id { |
| 256 | t.Fatalf("completion = %q, %v", raw, err) |
| 257 | } |
| 258 | if _, err := a.participation.UpdateID(); !os.IsNotExist(err) { |
| 259 | t.Fatalf("update record after settlement: %v", err) |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | // unfinishedGuard makes the fixture's transaction one that never changed |
| 264 | // the box: the signed reservation names another image as its target. |
| 265 | func unfinishedGuard(t *testing.T, f *applyFixture, id string) { |
| 266 | t.Helper() |
| 267 | path := filepath.Join(f.a.floor.Dir, "activation.json") |
| 268 | raw, err := os.ReadFile(path) |
| 269 | if err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | var tx guardTransaction |
| 273 | if err := json.Unmarshal(raw, &tx); err != nil { |
| 274 | t.Fatal(err) |
| 275 | } |
| 276 | var roster hostactors.Roster |
| 277 | if err := release.VerifyRootEnvelope(f.pub, tx.Roster, &roster); err != nil { |
| 278 | t.Fatal(err) |
| 279 | } |
| 280 | h := roster.Activations[id] |
| 281 | h.Hashes = map[string][]string{tx.Actor: {h.Hashes[tx.Actor][0], strings.Repeat("d", 64), h.Hashes[tx.Actor][2]}} |
| 282 | roster.Activations[id] = h |
| 283 | if err := roster.Validate("bucket"); err != nil { |
| 284 | t.Fatal(err) |
| 285 | } |
| 286 | if tx.Roster, err = release.SignRootEnvelope(release.RootSigner(f.priv), roster); err != nil { |
| 287 | t.Fatal(err) |
| 288 | } |
| 289 | raw, _ = json.Marshal(tx) |
| 290 | if err := hostfloor.Durable(path, raw, 0644); err != nil { |
| 291 | t.Fatal(err) |
| 292 | } |
| 293 | if finished, err := f.a.guardFinished(tx); err != nil || finished { |
| 294 | t.Fatalf("fixture transaction finished=%t err=%v, want unfinished", finished, err) |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | func TestRecoverDefersACensusRefusalOfAnUnfinishedTransaction(t *testing.T) { |
| 299 | f, id := guardRecoveryFixture(t) |
| 300 | a := f.a |
| 301 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 302 | t.Fatal(err) |
| 303 | } |
| 304 | unfinishedGuard(t, f, id) |
| 305 | refuseCensus(t, f) |
| 306 | if err := a.Run(t.Context(), true); err != nil { |
| 307 | t.Fatalf("recover = %v, want a deferral that exits 0", err) |
| 308 | } |
| 309 | // The withdrawal removes nothing until its census passes. |
| 310 | openTransaction(t, f, id, true) |
| 311 | if _, err := os.Stat(filepath.Join(a.floor.Dir, "completed.json")); !os.IsNotExist(err) { |
| 312 | t.Fatalf("an unfinished transaction recorded a completion: %v", err) |
| 313 | } |
| 314 | a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil } |
| 315 | if err := a.Run(t.Context(), true); err != nil { |
| 316 | t.Fatal(err) |
| 317 | } |
| 318 | for _, name := range []string{"update-begun.json", "activation.json", "completed.json"} { |
| 319 | if _, err := os.Stat(filepath.Join(a.floor.Dir, name)); !os.IsNotExist(err) { |
| 320 | t.Fatalf("%s after the withdrawal: %v", name, err) |
| 321 | } |
| 322 | } |
| 323 | } |
| 324 | |
| 325 | func TestRecoverDefersACensusRefusalOfAnAlreadyCompletedTransaction(t *testing.T) { |
| 326 | // Completion recorded and the marker gone, root's update record still |
| 327 | // open: only EndParticipation is left, and its census refuses. |
| 328 | f, id := guardRecoveryFixture(t) |
| 329 | a := f.a |
| 330 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 331 | t.Fatal(err) |
| 332 | } |
| 333 | if err := hostfloor.Durable(filepath.Join(a.floor.Dir, "completed.json"), []byte(id), 0644); err != nil { |
| 334 | t.Fatal(err) |
| 335 | } |
| 336 | if err := hostfloor.RemoveDurable(filepath.Join(a.floor.Dir, "update-begun.json")); err != nil { |
| 337 | t.Fatal(err) |
| 338 | } |
| 339 | calls := refuseCensus(t, f) |
| 340 | if err := a.Run(t.Context(), true); err != nil { |
| 341 | t.Fatalf("recover = %v, want a deferral that exits 0", err) |
| 342 | } |
| 343 | if *calls != 1 { |
| 344 | t.Fatalf("census ran %d times, want 1", *calls) |
| 345 | } |
| 346 | openTransaction(t, f, id, false) |
| 347 | if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id { |
| 348 | t.Fatalf("completion = %q, %v", raw, err) |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | func TestRecoverStillFailsSettlementErrorsThatAreNotCensusRefusals(t *testing.T) { |
| 353 | for _, tc := range []struct { |
| 354 | name string |
| 355 | setup func(*testing.T, *applyFixture, string) |
| 356 | }{ |
| 357 | {"malformed-ledger-state", func(t *testing.T, f *applyFixture, _ string) { |
| 358 | // The verifier refuses the ledger itself, not through the census. |
| 359 | f.a.participation.VerifyLaunches = func(context.Context, uint64) error { return rootparticipation.ErrUnavailable } |
| 360 | old := recoverPendingControllers |
| 361 | t.Cleanup(func() { recoverPendingControllers = old }) |
| 362 | recoverPendingControllers = func(context.Context, string, string) error { return nil } |
| 363 | }}, |
| 364 | {"live-controller", func(t *testing.T, f *applyFixture, _ string) { |
| 365 | if err := os.WriteFile(filepath.Join(f.a.dataDir, hostexecDir, "host.json"), []byte(`{"state":"pending"}`), 0600); err != nil { |
| 366 | t.Fatal(err) |
| 367 | } |
| 368 | old := recoverPendingControllers |
| 369 | t.Cleanup(func() { recoverPendingControllers = old }) |
| 370 | recoverPendingControllers = func(context.Context, string, string) error { return nil } |
| 371 | }}, |
| 372 | {"image-verification", func(t *testing.T, f *applyFixture, _ string) { |
| 373 | if err := os.WriteFile(systemBinaryPath(f.a.optDir), []byte("an unsigned image"), 0755); err != nil { |
| 374 | t.Fatal(err) |
| 375 | } |
| 376 | }}, |
| 377 | {"marker-for-another-transaction", func(t *testing.T, f *applyFixture, _ string) { |
| 378 | if err := hostfloor.Durable(filepath.Join(f.a.floor.Dir, "update-begun.json"), []byte(`{"id":"`+strings.Repeat("e", 32)+`"}`), 0644); err != nil { |
| 379 | t.Fatal(err) |
| 380 | } |
| 381 | }}, |
| 382 | } { |
| 383 | t.Run(tc.name, func(t *testing.T) { |
| 384 | f, id := guardRecoveryFixture(t) |
| 385 | if err := f.a.floor.Begin(t.Context(), id, f.a.dataDir); err != nil { |
| 386 | t.Fatal(err) |
| 387 | } |
| 388 | tc.setup(t, f, id) |
| 389 | err := f.a.Run(t.Context(), true) |
| 390 | if err == nil || errors.Is(err, errSettlementDeferred) { |
| 391 | t.Fatalf("recover = %v, want the settlement's own failure", err) |
| 392 | } |
| 393 | if _, statErr := os.Stat(filepath.Join(f.a.floor.Dir, "activation.json")); statErr != nil { |
| 394 | t.Fatalf("reservation discarded: %v", statErr) |
| 395 | } |
| 396 | }) |
| 397 | } |
| 398 | } |
| 399 | |
| 400 | func TestOnlyCensusRefusal(t *testing.T) { |
| 401 | refusal := harnessRefusal() |
| 402 | for _, tc := range []struct { |
| 403 | err error |
| 404 | want bool |
| 405 | }{ |
| 406 | {refusal, true}, |
| 407 | {fmt.Errorf("end participation: %w", refusal), true}, |
| 408 | {errors.Join(refusal, nil), true}, |
| 409 | {errors.Join(refusal, harnessRefusal()), true}, |
| 410 | {errors.Join(refusal, errors.New("remove update.json: read-only file system")), false}, |
| 411 | {fmt.Errorf("x: %w", errors.Join(refusal, os.ErrPermission)), false}, |
| 412 | {os.ErrPermission, false}, |
| 413 | {rootparticipation.ErrUnavailable, false}, |
| 414 | {nil, false}, |
| 415 | } { |
| 416 | if got := onlyCensusRefusal(tc.err); got != tc.want { |
| 417 | t.Fatalf("onlyCensusRefusal(%v) = %t, want %t", tc.err, got, tc.want) |
| 418 | } |
| 419 | } |
| 420 | } |
| 421 | |
| 422 | func TestRecoverWithoutAnArmedFloorKeepsALateRefusalNonFatal(t *testing.T) { |
| 423 | // The controllers' census passes; settling the update record at the end |
| 424 | // of the run (settleUpdateBegun -> EndUpdateWith) refuses. As on main |
| 425 | // that is logged, not fatal, and the record stays open. |
| 426 | f := newApplyFixture(t, "stable", "old") |
| 427 | if err := f.a.beginExecutableChange(t.Context(), "new", f.now); err != nil { |
| 428 | t.Fatal(err) |
| 429 | } |
| 430 | calls := refuseCensusFrom(t, f, 2) |
| 431 | if err := f.a.Run(t.Context(), true); err != nil { |
| 432 | t.Fatalf("recover = %v", err) |
| 433 | } |
| 434 | if *calls < 2 { |
| 435 | t.Fatalf("census ran %d times; the late settlement never ran", *calls) |
| 436 | } |
| 437 | if _, err := f.a.participation.UpdateID(); err != nil { |
| 438 | t.Fatalf("a refused settlement closed the update record: %v", err) |
| 439 | } |
| 440 | } |
| 441 | |
| 442 | func TestRecoverHoldsAcquisitionsWhenNoUpdateRecordWasBegun(t *testing.T) { |
| 443 | // A prepared transaction: the reservation persisted, root's marker and |
| 444 | // update record never written. The census refuses the settlement's |
| 445 | // BeginUpdate, which withdraws its own record; recover must still |
| 446 | // leave acquisitions refused. |
| 447 | for _, finished := range []bool{true, false} { |
| 448 | t.Run(map[bool]string{true: "finished", false: "unfinished"}[finished], func(t *testing.T) { |
| 449 | f, id := guardRecoveryFixture(t) |
| 450 | a := f.a |
| 451 | if !finished { |
| 452 | unfinishedGuard(t, f, id) |
| 453 | } |
| 454 | if _, err := a.participation.UpdateID(); !os.IsNotExist(err) { |
| 455 | t.Fatalf("fixture has an update record: %v", err) |
| 456 | } |
| 457 | refuseCensus(t, f) |
| 458 | if err := a.Run(t.Context(), true); err != nil { |
| 459 | t.Fatalf("recover = %v, want a deferral that exits 0", err) |
| 460 | } |
| 461 | openTransaction(t, f, id, false) |
| 462 | a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil } |
| 463 | if err := a.Run(t.Context(), true); err != nil { |
| 464 | t.Fatal(err) |
| 465 | } |
| 466 | if _, err := a.participation.UpdateID(); !os.IsNotExist(err) { |
| 467 | t.Fatalf("barrier left after settlement: %v", err) |
| 468 | } |
| 469 | if err := a.participation.Acquire(t.Context(), func() error { return nil }); err != nil { |
| 470 | t.Fatalf("acquisition after settlement = %v", err) |
| 471 | } |
| 472 | }) |
| 473 | } |
| 474 | } |
| 475 | |
| 476 | func TestRecoverReplaysASettledTransactionWithoutACensus(t *testing.T) { |
| 477 | // Completion recorded, marker and update record gone: settled. A |
| 478 | // replay closes nothing, runs no census and blocks nothing. |
| 479 | f, id := guardRecoveryFixture(t) |
| 480 | a := f.a |
| 481 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 482 | t.Fatal(err) |
| 483 | } |
| 484 | if err := a.Run(t.Context(), true); err != nil { |
| 485 | t.Fatal(err) |
| 486 | } |
| 487 | if raw, err := os.ReadFile(filepath.Join(a.floor.Dir, "completed.json")); err != nil || string(raw) != id { |
| 488 | t.Fatalf("fixture not settled: %q %v", raw, err) |
| 489 | } |
| 490 | calls := refuseCensus(t, f) |
| 491 | if err := a.Run(t.Context(), true); err != nil { |
| 492 | t.Fatalf("replay = %v", err) |
| 493 | } |
| 494 | if *calls != 0 { |
| 495 | t.Fatalf("a settled replay ran the census %d times", *calls) |
| 496 | } |
| 497 | if _, err := a.participation.UpdateID(); !os.IsNotExist(err) { |
| 498 | t.Fatalf("a settled replay left an update record: %v", err) |
| 499 | } |
| 500 | } |
| 501 | |
| 502 | // journalOpen writes an apply's journal for a recovery to finish. |
| 503 | func journalOpen(t *testing.T, f *applyFixture, j systemJournal) { |
| 504 | t.Helper() |
| 505 | j.StartedAt = f.now |
| 506 | if err := f.a.journal(j); err != nil { |
| 507 | t.Fatal(err) |
| 508 | } |
| 509 | } |
| 510 | |
| 511 | func TestRecoverDefersARefusalAfterTheJournalIsRemoved(t *testing.T) { |
| 512 | // Recovery's own census passes; the settlement after the journal is |
| 513 | // durably removed refuses. Unarmed forward (from gated) and backward |
| 514 | // (from installed); guarded through the journal close a guarded |
| 515 | // fixture can run, the legacy backing-up journal (the guarded forward |
| 516 | // and backward restores run a guarded image, which it cannot; the |
| 517 | // guarded backward close is TestGuardedRollbackCloseDefers...). |
| 518 | for _, tc := range []struct { |
| 519 | name string |
| 520 | guarded bool |
| 521 | forward bool |
| 522 | }{{"guarded-backing-up", true, true}, {"unarmed-forward", false, true}, {"unarmed-backward", false, false}} { |
| 523 | t.Run(tc.name, func(t *testing.T) { |
| 524 | var f *applyFixture |
| 525 | id := "" |
| 526 | if tc.guarded { |
| 527 | f, id = guardRecoveryFixture(t) |
| 528 | if err := f.a.floor.Begin(t.Context(), id, f.a.dataDir); err != nil { |
| 529 | t.Fatal(err) |
| 530 | } |
| 531 | } else { |
| 532 | f = newApplyFixture(t, "stable", "old-build") |
| 533 | if err := f.a.beginExecutableChange(t.Context(), "new-build", f.now); err != nil { |
| 534 | t.Fatal(err) |
| 535 | } |
| 536 | id, _ = f.a.participation.UpdateID() |
| 537 | } |
| 538 | a := f.a |
| 539 | // carlos.prev is the same signed build: the guard verifies it |
| 540 | // before a recovery with a journal open. |
| 541 | body, _ := os.ReadFile(systemBinaryPath(a.optDir)) |
| 542 | if err := os.WriteFile(systemPrevPath(a.optDir), body, 0o755); err != nil { |
| 543 | t.Fatal(err) |
| 544 | } |
| 545 | prev := systemPrevPath(a.optDir) |
| 546 | f.answer["version:"+prev] = "carlos old-build\n" |
| 547 | f.answer["plan:"+prev] = f.answer["plan:"+systemBinaryPath(a.optDir)] |
| 548 | f.answer["units:"+prev] = f.answer["units:"+systemBinaryPath(a.optDir)] |
| 549 | if tc.guarded { |
| 550 | journalOpen(t, f, systemJournal{Phase: systemPhaseBackingUp, Target: "new-build", PrevBuild: "old-build"}) |
| 551 | } else if tc.forward { |
| 552 | journalOpen(t, f, systemJournal{Phase: systemPhaseGated, Target: "old-build", PrevBuild: "old-build"}) |
| 553 | } else { |
| 554 | journalOpen(t, f, systemJournal{Phase: systemPhaseInstalled, Target: "new-build", PrevBuild: "old-build"}) |
| 555 | } |
| 556 | old := recoverPendingControllers |
| 557 | t.Cleanup(func() { recoverPendingControllers = old }) |
| 558 | recoverPendingControllers = func(context.Context, string, string) error { return nil } |
| 559 | removed := false |
| 560 | a.participation.VerifyLaunches = func(context.Context, uint64) error { |
| 561 | if j, _ := readSystemJournal(a.optDir); j == nil { |
| 562 | removed = true |
| 563 | return harnessRefusal() |
| 564 | } |
| 565 | return nil |
| 566 | } |
| 567 | if err := a.Run(t.Context(), true); err != nil { |
| 568 | t.Fatalf("recover = %v, want a deferral that exits 0", err) |
| 569 | } |
| 570 | if !removed { |
| 571 | t.Fatal("the census never ran after the journal was removed") |
| 572 | } |
| 573 | if j, _ := readSystemJournal(a.optDir); j != nil { |
| 574 | t.Fatal("journal kept") |
| 575 | } |
| 576 | want := systemOutcomeRolledBack |
| 577 | if tc.forward { |
| 578 | want = systemOutcomeApplied |
| 579 | } |
| 580 | if r := f.result(); !tc.guarded && (r == nil || r.Outcome != want) { |
| 581 | t.Fatalf("result = %+v, want %s", r, want) |
| 582 | } |
| 583 | if got, err := a.participation.UpdateID(); err != nil || got != id { |
| 584 | t.Fatalf("update record = %q, %v; want %s held", got, err, id) |
| 585 | } |
| 586 | if err := a.participation.Acquire(t.Context(), func() error { return nil }); err == nil { |
| 587 | t.Fatal("a claim acquisition started while the settlement was deferred") |
| 588 | } |
| 589 | // The next recover, with the census passing, settles it. |
| 590 | a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil } |
| 591 | if err := a.Run(t.Context(), true); err != nil { |
| 592 | t.Fatal(err) |
| 593 | } |
| 594 | if _, err := a.participation.UpdateID(); !os.IsNotExist(err) { |
| 595 | t.Fatalf("update record after settlement: %v", err) |
| 596 | } |
| 597 | }) |
| 598 | } |
| 599 | } |
| 600 | |
| 601 | func TestRecoverStillFailsANonCensusErrorAfterTheJournalIsRemoved(t *testing.T) { |
| 602 | f, id := guardRecoveryFixture(t) |
| 603 | a := f.a |
| 604 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 605 | t.Fatal(err) |
| 606 | } |
| 607 | journalOpen(t, f, systemJournal{Phase: systemPhaseGated, Target: "old-build", PrevBuild: "old-build"}) |
| 608 | a.participation.VerifyLaunches = func(context.Context, uint64) error { |
| 609 | if j, _ := readSystemJournal(a.optDir); j == nil { |
| 610 | return rootparticipation.ErrUnavailable |
| 611 | } |
| 612 | return nil |
| 613 | } |
| 614 | err := a.Run(t.Context(), true) |
| 615 | if err == nil || errors.Is(err, errSettlementDeferred) { |
| 616 | t.Fatalf("recover = %v, want the settlement's own failure", err) |
| 617 | } |
| 618 | } |
| 619 | |
| 620 | func TestGuardedRollbackCloseDefersARefusalAfterTheJournalIsRemoved(t *testing.T) { |
| 621 | // The backward close of a guarded transaction (closeGuardRollback, as |
| 622 | // recovery's rollback and the apply's rollback call it): the rollback is |
| 623 | // recorded, the journal removed, and the settlement's census refuses. |
| 624 | // Restoring carlos.prev runs a guarded image, which this fixture |
| 625 | // cannot, so the close is driven directly; Run's consumption of the |
| 626 | // deferral is the forward case's. |
| 627 | f, id := guardRecoveryFixture(t) |
| 628 | a := f.a |
| 629 | if err := a.floor.Begin(t.Context(), id, a.dataDir); err != nil { |
| 630 | t.Fatal(err) |
| 631 | } |
| 632 | journalOpen(t, f, systemJournal{Phase: systemPhaseInstalled, Target: "new-build", PrevBuild: "old-build"}) |
| 633 | refuseCensus(t, f) |
| 634 | err := a.closeGuardRollback(t.Context()) |
| 635 | if !errors.Is(err, errSettlementDeferred) || !a.deferSettlement(true, err) { |
| 636 | t.Fatalf("rollback close = %v, want a deferral", err) |
| 637 | } |
| 638 | if a.deferSettlement(false, err) { |
| 639 | t.Fatal("the apply consumed a deferral") |
| 640 | } |
| 641 | if j, _ := readSystemJournal(a.optDir); j != nil { |
| 642 | t.Fatal("journal kept") |
| 643 | } |
| 644 | if begun, err := hostfloor.ReadBegun(a.floor.Dir); err != nil || !begun.RolledBack { |
| 645 | t.Fatalf("rollback not recorded: %+v %v", begun, err) |
| 646 | } |
| 647 | openTransaction(t, f, id, true) |
| 648 | } |
| 649 | |
| 650 | func TestRecoverDoesNotDeferWithoutABarrier(t *testing.T) { |
| 651 | // A barrier that cannot be written (here an id the ledger refuses) |
| 652 | // leaves the census refusal fatal: recover never exits 0 with |
| 653 | // acquisitions open. |
| 654 | f, _ := guardRecoveryFixture(t) |
| 655 | err := f.a.settleWithoutJournal(t.Context(), "bad id", harnessRefusal()) |
| 656 | if err == nil || errors.Is(err, errSettlementDeferred) || !strings.Contains(err.Error(), "acquisition barrier") { |
| 657 | t.Fatalf("settlement without a barrier = %v", err) |
| 658 | } |
| 659 | if err := f.a.participation.Acquire(t.Context(), func() error { return nil }); err != nil { |
| 660 | t.Fatalf("fixture: %v", err) |
| 661 | } |
| 662 | } |
| 663 | |
| 664 | // failingBarrierStore is the participation store with one write refused: |
| 665 | // the barrier's update.json, written after BeginUpdate's own write of it |
| 666 | // was withdrawn by a census refusal. |
| 667 | type failingBarrierStore struct { |
| 668 | rootstate.Store |
| 669 | writes int |
| 670 | } |
| 671 | |
| 672 | func (f *failingBarrierStore) Write(name string, raw []byte) error { |
| 673 | if name == "update.json" { |
| 674 | f.writes++ |
| 675 | if f.writes == 2 { |
| 676 | return errors.New("barrier write refused for the test") |
| 677 | } |
| 678 | } |
| 679 | return f.Store.Write(name, raw) |
| 680 | } |
| 681 | |
| 682 | // legacyMarker writes the update-begun marker alone: root participation, |
| 683 | // no armed floor, no update record, no journal (Astra round 3). |
| 684 | func legacyMarker(t *testing.T, f *applyFixture) { |
| 685 | t.Helper() |
| 686 | a := f.a |
| 687 | if a.guarded() { |
| 688 | t.Fatal("fixture has an armed floor") |
| 689 | } |
| 690 | if err := os.MkdirAll(systemGuardDir(a.optDir), 0o755); err != nil { |
| 691 | t.Fatal(err) |
| 692 | } |
| 693 | if err := writeJSONDurable(updateBegunPath(a.optDir), updateBegun{Target: "new", At: f.now}, 0o644); err != nil { |
| 694 | t.Fatal(err) |
| 695 | } |
| 696 | if _, err := a.participation.UpdateID(); !os.IsNotExist(err) { |
| 697 | t.Fatalf("fixture has an update record: %v", err) |
| 698 | } |
| 699 | if j, err := readSystemJournal(a.optDir); err != nil || j != nil { |
| 700 | t.Fatalf("fixture has a journal: %v %v", j, err) |
| 701 | } |
| 702 | } |
| 703 | |
| 704 | func TestRecoverHoldsAcquisitionsForALegacyMarker(t *testing.T) { |
| 705 | // checkUnarmedParticipationRecovery sees no update record and no |
| 706 | // journal and returns; the run's end settles the marker through |
| 707 | // clearUpdateBegun, whose BeginUpdate the census refuses (and which |
| 708 | // withdraws the record it wrote). Recover must exit 0 with the marker |
| 709 | // kept and acquisitions held refused, and settle both once the census |
| 710 | // passes. |
| 711 | f := newApplyFixture(t, "stable", "old") |
| 712 | a := f.a |
| 713 | legacyMarker(t, f) |
| 714 | refuseCensus(t, f) |
| 715 | if err := a.Run(t.Context(), true); err != nil { |
| 716 | t.Fatalf("recover = %v, want a deferral that exits 0", err) |
| 717 | } |
| 718 | if _, err := os.Lstat(updateBegunPath(a.optDir)); err != nil { |
| 719 | t.Fatalf("the marker went without a settlement: %v", err) |
| 720 | } |
| 721 | if id, err := a.participation.UpdateID(); err != nil || id != "recover-legacy" { |
| 722 | t.Fatalf("barrier = %q, %v; want root's update record under recover-legacy", id, err) |
| 723 | } |
| 724 | if err := a.participation.Acquire(t.Context(), func() error { return nil }); err == nil { |
| 725 | t.Fatal("an acquisition passed while recover had deferred the legacy marker's settlement") |
| 726 | } |
| 727 | a.participation.VerifyLaunches = func(context.Context, uint64) error { return nil } |
| 728 | if err := a.Run(t.Context(), true); err != nil { |
| 729 | t.Fatal(err) |
| 730 | } |
| 731 | if _, err := os.Lstat(updateBegunPath(a.optDir)); !os.IsNotExist(err) { |
| 732 | t.Fatalf("marker left after settlement: %v", err) |
| 733 | } |
| 734 | if _, err := a.participation.UpdateID(); !os.IsNotExist(err) { |
| 735 | t.Fatalf("barrier left after settlement: %v", err) |
| 736 | } |
| 737 | if err := a.participation.Acquire(t.Context(), func() error { return nil }); err != nil { |
| 738 | t.Fatalf("acquisition after settlement = %v", err) |
| 739 | } |
| 740 | } |
| 741 | |
| 742 | func TestRecoverFailsALegacyMarkerWhoseBarrierCannotBeWritten(t *testing.T) { |
| 743 | // The same start, with the barrier's write refused: recover must fail, |
| 744 | // not exit 0 with acquisitions open; the marker stays for the next run. |
| 745 | f := newApplyFixture(t, "stable", "old") |
| 746 | a := f.a |
| 747 | legacyMarker(t, f) |
| 748 | refuseCensus(t, f) |
| 749 | a.participation.State = &failingBarrierStore{Store: a.participation.State} |
| 750 | err := a.Run(t.Context(), true) |
| 751 | if err == nil || !errors.Is(err, errAcquisitionBarrier) { |
| 752 | t.Fatalf("recover with no barrier = %v, want it fatal", err) |
| 753 | } |
| 754 | if _, err := os.Lstat(updateBegunPath(a.optDir)); err != nil { |
| 755 | t.Fatalf("the marker went without a settlement: %v", err) |
| 756 | } |
| 757 | } |
| 758 | |
| 759 | // prerequisiteFailingStore is the participation store whose census stops |
| 760 | // reading once BeginUpdate has withdrawn the update record it wrote: the |
| 761 | // barrier's prerequisite, ParticipationRequired, then fails. |
| 762 | type prerequisiteFailingStore struct { |
| 763 | rootstate.Store |
| 764 | withdrawn bool |
| 765 | } |
| 766 | |
| 767 | func (p *prerequisiteFailingStore) Remove(name string) error { |
| 768 | if name == "update.json" { |
| 769 | p.withdrawn = true |
| 770 | } |
| 771 | return p.Store.Remove(name) |
| 772 | } |
| 773 | |
| 774 | func (p *prerequisiteFailingStore) Read(name string) ([]byte, error) { |
| 775 | if name == "census.json" && p.withdrawn { |
| 776 | return nil, errors.New("census read refused for the test") |
| 777 | } |
| 778 | return p.Store.Read(name) |
| 779 | } |
| 780 | |
| 781 | func TestRecoverFailsALegacyMarkerWhoseBarrierPrerequisiteFails(t *testing.T) { |
| 782 | // Astra round 4: after the census refused BeginUpdate and the record it |
| 783 | // wrote was withdrawn, the check whether a barrier is needed fails. That |
| 784 | // is a barrier failure: recover must not log it and exit 0. |
| 785 | f := newApplyFixture(t, "stable", "old") |
| 786 | a := f.a |
| 787 | legacyMarker(t, f) |
| 788 | refuseCensus(t, f) |
| 789 | a.participation.State = &prerequisiteFailingStore{Store: a.participation.State} |
| 790 | err := a.Run(t.Context(), true) |
| 791 | if err == nil || !errors.Is(err, errAcquisitionBarrier) { |
| 792 | t.Fatalf("recover with the barrier prerequisite failing = %v, want it fatal", err) |
| 793 | } |
| 794 | if _, err := os.Lstat(updateBegunPath(a.optDir)); err != nil { |
| 795 | t.Fatalf("the marker went without a settlement: %v", err) |
| 796 | } |
| 797 | } |
| 798 | |
amadan