| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "testing" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/registry" |
| 10 | "github.com/carlosframework/platform/internal/release" |
| 11 | ) |
| 12 | |
| 13 | type e8ExeUnits struct{ *plan14RestartUnits } |
| 14 | |
| 15 | func (u e8ExeUnits) Exe(pid int) (string, error) { |
| 16 | if len(u.effects) > 0 { |
| 17 | return "", os.ErrPermission |
| 18 | } |
| 19 | return u.plan14RestartUnits.Exe(pid) |
| 20 | } |
| 21 | |
| 22 | func TestPlan14ERecoveryRestartOutcome(t *testing.T) { |
| 23 | for _, fault := range []string{"binding", "final binding", "exe", "failed", "auto-restart", "not ready"} { |
| 24 | t.Run(fault, func(t *testing.T) { |
| 25 | e, u, c, source, pub := signedRecoveryFixture(t) |
| 26 | if err := e.root.reg.SetVersions(observedHost, "v1", ""); err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | u.afterRestart = func() { u.props.NRestarts = 0 } |
| 30 | u.afterEffect = func() { |
| 31 | if len(u.effects) != 1 { |
| 32 | return |
| 33 | } |
| 34 | if fault == "failed" { |
| 35 | u.props.ActiveState, u.props.MainPID = "failed", 0 |
| 36 | } |
| 37 | if fault == "auto-restart" { |
| 38 | u.props.ActiveState, u.props.SubState, u.props.NRestarts = "activating", "auto-restart", 1 |
| 39 | } |
| 40 | } |
| 41 | c.legacy = func() (bool, error) { return true, nil } |
| 42 | c.legacyReady = func(registry.Route) func() error { |
| 43 | return func() error { |
| 44 | if fault == "not ready" && len(u.effects) == 1 { |
| 45 | return os.ErrDeadlineExceeded |
| 46 | } |
| 47 | return nil |
| 48 | } |
| 49 | } |
| 50 | orig := c.host |
| 51 | calls := 0 |
| 52 | c.host = func(ctx context.Context, host string) (proofBinding, error) { |
| 53 | if len(u.effects) == 1 { |
| 54 | calls++ |
| 55 | if fault != "final binding" || calls >= 2 { |
| 56 | if fault == "failed" { |
| 57 | u.props.ActiveState, u.props.MainPID = "failed", 0 |
| 58 | } |
| 59 | if fault == "auto-restart" { |
| 60 | u.props.ActiveState, u.props.SubState, u.props.NRestarts = "activating", "auto-restart", 1 |
| 61 | } |
| 62 | return proofBinding{}, os.ErrPermission |
| 63 | } |
| 64 | } |
| 65 | return orig(ctx, host) |
| 66 | } |
| 67 | if fault == "exe" { |
| 68 | source.sys = e8ExeUnits{u} |
| 69 | } |
| 70 | n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink} |
| 71 | fallbacks := 0 |
| 72 | err := release.Pass(t.Context(), e.root.reg, e.store, pub, u.opt, func(host string) error { |
| 73 | rt, e := e.root.reg.Get(host) |
| 74 | if e != nil { |
| 75 | return e |
| 76 | } |
| 77 | fallbacks++ |
| 78 | return legacyCheckedUnitRestart(source.sys, u.opt, func(string) string { return "5s" }, *rt, nil, c) |
| 79 | }, release.BakeDir(e.bake), release.WithRestartFailurePropagation(), release.WithRecovery(func(ctx context.Context, rt registry.Route) (bool, error) { |
| 80 | return n.recover(ctx, rt.Host, c, func(string) string { return "5s" }, e.root.reg) |
| 81 | }, c.allowLegacy), release.WithRollbackRestart(func(host string) error { |
| 82 | rt, err := e.root.reg.Get(host) |
| 83 | if err != nil { |
| 84 | return err |
| 85 | } |
| 86 | return legacyRollbackRestart(source.sys, u.opt, func(string) string { return "5s" }, *rt, c) |
| 87 | })) |
| 88 | failed := fault == "failed" || fault == "auto-restart" || fault == "not ready" |
| 89 | want, count := "v2", 1 |
| 90 | if failed { |
| 91 | want, count = "v1", 3 |
| 92 | } |
| 93 | live, liveErr := release.LiveVersion(u.opt, observedHost) |
| 94 | if liveErr != nil || live != want || len(u.effects) != count || fallbacks != 1 || (err != nil) != failed { |
| 95 | t.Fatalf("outcome: live=%s/%v effects=%v fallback=%d err=%v", live, liveErr, u.effects, fallbacks, err) |
| 96 | } |
| 97 | stamp, stampErr := e.root.reg.Get(observedHost) |
| 98 | if stampErr != nil || stamp.Version != want { |
| 99 | t.Fatalf("accepted outcome not stamped: %+v %v", stamp, stampErr) |
| 100 | } |
| 101 | disk, _ := c.attempts() |
| 102 | if names, err := disk.Names(); err != nil || len(names) != 0 { |
| 103 | t.Fatalf("no-census recovery created authority: %v %v", names, err) |
| 104 | } |
| 105 | if calls != 0 { |
| 106 | t.Fatalf("recovery selected binding: %d", calls) |
| 107 | } |
| 108 | }) |
| 109 | } |
| 110 | } |
| 111 | |
| 112 | func TestPlan14ERecoveryNotUpPrecedesBinding(t *testing.T) { |
| 113 | err := errors.Join(errRestartBinding, &release.UnitNotUpError{Unit: "console.service", Reason: "failed"}) |
| 114 | if !restartEffectFailed(err) { |
| 115 | t.Fatal("binding masked real not-up result") |
| 116 | } |
| 117 | } |
| 118 | |
| 119 | func TestPlan14ERecoveryStrictNotUpPrecedesBinding(t *testing.T) { |
| 120 | e, u, c, source, pub := signedRecoveryFixture(t) |
| 121 | original := c.host |
| 122 | c.host = func(ctx context.Context, host string) (proofBinding, error) { |
| 123 | if len(u.effects) == 1 { |
| 124 | u.props.ActiveState, u.props.MainPID = "failed", 0 |
| 125 | return proofBinding{}, os.ErrPermission |
| 126 | } |
| 127 | return original(ctx, host) |
| 128 | } |
| 129 | n := namedUnitRecovery{source: source, pub: pub, readLive: os.Readlink} |
| 130 | handled, err := n.recover(t.Context(), observedHost, c, func(string) string { return "5s" }, e.root.reg) |
| 131 | var notUp *release.UnitNotUpError |
| 132 | if !handled || !errors.As(err, ¬Up) || len(u.effects) != 3 { |
| 133 | t.Fatalf("strict outcome lost: handled=%v err=%v effects=%v", handled, err, u.effects) |
| 134 | } |
| 135 | } |
| 136 | |
amadan