| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "sync" |
| 8 | "testing" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/hostwrite" |
| 11 | "github.com/carlosframework/platform/internal/release" |
| 12 | ) |
| 13 | |
| 14 | func TestPlan14RollbackCrashResume(t *testing.T) { |
| 15 | t.Parallel() |
| 16 | for _, boundary := range []string{"forward restart", "rollback pending", "rollback running", "operator stopped", "changed link"} { |
| 17 | t.Run(boundary, func(t *testing.T) { |
| 18 | e, rt, u, c, pub := plan14AdoptionFixture(t) |
| 19 | u.crashNew = true |
| 20 | forward := rootRestartRouteWith(e.root.reg, u, u.opt, func(string) string { return "5s" }, nil, c) |
| 21 | crashed := errors.New("root process exited") |
| 22 | func() { |
| 23 | defer func() { |
| 24 | if got := recover(); got != crashed { |
| 25 | t.Fatalf("crash boundary: %v", got) |
| 26 | } |
| 27 | }() |
| 28 | _ = release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, func(host string) error { |
| 29 | if err := forward(host); err == nil { |
| 30 | t.Fatal("failed update succeeded") |
| 31 | } |
| 32 | panic(crashed) |
| 33 | }, release.BakeDir(e.bake), release.WithRestartFailurePropagation()) |
| 34 | }() |
| 35 | s, err := c.attempts() |
| 36 | if err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | a, err := readRollbackAttempt(s, rt.Host) |
| 40 | if err != nil { |
| 41 | t.Fatal(err) |
| 42 | } |
| 43 | if boundary == "rollback pending" || boundary == "rollback running" { |
| 44 | a.Recovering = true |
| 45 | if err := writeRollbackState(s, rollbackJournalKey(rt.Host), a); err != nil { |
| 46 | t.Fatal(err) |
| 47 | } |
| 48 | if err := release.PointLive(u.opt, rt.Host, rt.Account, "v1", rt.App); err != nil { |
| 49 | t.Fatal(err) |
| 50 | } |
| 51 | if boundary == "rollback running" { |
| 52 | if err := u.Restart(rt.Unit); err != nil { |
| 53 | t.Fatal(err) |
| 54 | } |
| 55 | } |
| 56 | } |
| 57 | if boundary == "operator stopped" { |
| 58 | u.props.ActiveState, u.props.SubState = "inactive", "dead" |
| 59 | } |
| 60 | if boundary == "changed link" { |
| 61 | if err := os.Remove(release.LiveLink(u.opt, rt.Host)); err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | } |
| 65 | before := len(u.effects) |
| 66 | c.started = &sync.Map{} |
| 67 | err = recoverRootRestarts(context.Background(), u, u.opt, func(string) string { return "5s" }, c, hostwrite.Config{}) |
| 68 | if boundary == "operator stopped" || boundary == "changed link" { |
| 69 | if err == nil || len(u.effects) != before { |
| 70 | t.Fatalf("unsafe resume: %v, %v", err, u.effects) |
| 71 | } |
| 72 | return |
| 73 | } |
| 74 | if err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | if u.exe != release.VersionPath(u.opt, rt.Account, "v1", rt.App) || u.props.MainPID <= 0 { |
| 78 | t.Fatalf("resume did not restore the old process: %+v", u) |
| 79 | } |
| 80 | if boundary == "rollback running" && len(u.effects) != before { |
| 81 | t.Fatal("resumed verification restarted an already recovered unit") |
| 82 | } |
| 83 | a, err = readRollbackAttempt(s, rt.Host) |
| 84 | if err != nil || !a.Settled { |
| 85 | t.Fatalf("resume did not settle: %+v, %v", a, err) |
| 86 | } |
| 87 | if _, err := s.Read(rollbackCapabilityKey(rt.Host, a.ID)); !errors.Is(err, os.ErrNotExist) { |
| 88 | t.Fatal("resume retained capability") |
| 89 | } |
| 90 | }) |
| 91 | } |
| 92 | } |
| 93 | |
amadan