| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "reflect" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/activator" |
| 13 | "github.com/carlosframework/platform/internal/hostwrite" |
| 14 | "github.com/carlosframework/platform/internal/provision" |
| 15 | "github.com/carlosframework/platform/internal/registry" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | "github.com/carlosframework/platform/internal/rootstate" |
| 18 | ) |
| 19 | |
| 20 | // Each unit has separate process state. In particular, restarting the local |
| 21 | // binding must never look like a restart of a different box-local stamp. |
| 22 | type fix5Units struct { |
| 23 | units map[string]*r2Units |
| 24 | events []string |
| 25 | resetError, tryError bool |
| 26 | } |
| 27 | |
| 28 | func (u *fix5Units) Show(name string) (unitProps, error) { return u.units[name].Show(name) } |
| 29 | func (u *fix5Units) Exe(pid int) (string, error) { |
| 30 | for _, unit := range u.units { |
| 31 | if unit.props.MainPID == pid { |
| 32 | return unit.Exe(pid) |
| 33 | } |
| 34 | } |
| 35 | return "", os.ErrNotExist |
| 36 | } |
| 37 | func (u *fix5Units) StartTime(pid int) (string, error) { |
| 38 | for _, unit := range u.units { |
| 39 | if unit.props.MainPID == pid { |
| 40 | return unit.StartTime(pid) |
| 41 | } |
| 42 | } |
| 43 | return "", os.ErrNotExist |
| 44 | } |
| 45 | func (u *fix5Units) JournalTail(string, int) string { return "" } |
| 46 | func (u *fix5Units) TryRestart(name string) error { |
| 47 | u.events = append(u.events, "try-restart "+name) |
| 48 | if u.tryError { |
| 49 | u.tryError = false |
| 50 | return errors.New("try-restart: bus timeout") |
| 51 | } |
| 52 | return u.units[name].TryRestart(name) |
| 53 | } |
| 54 | func (u *fix5Units) Restart(name string) error { |
| 55 | u.events = append(u.events, "restart "+name) |
| 56 | return u.units[name].Restart(name) |
| 57 | } |
| 58 | func (u *fix5Units) ResetFailed(name string) error { |
| 59 | u.events = append(u.events, "reset-failed "+name) |
| 60 | if u.resetError { |
| 61 | return errors.New("reset-failed: bus timeout") |
| 62 | } |
| 63 | return u.units[name].ResetFailed(name) |
| 64 | } |
| 65 | func (u *fix5Units) Start(name string) error { |
| 66 | u.events = append(u.events, "start "+name) |
| 67 | return u.units[name].Start(name) |
| 68 | } |
| 69 | |
| 70 | type fix5Result struct { |
| 71 | Events []string |
| 72 | Live, Previous string |
| 73 | Route registry.Route |
| 74 | Processes map[string]string |
| 75 | } |
| 76 | |
| 77 | func fix5Run(t *testing.T, kind, history string, main bool) []fix5Result { |
| 78 | t.Helper() |
| 79 | sourceKind := kind |
| 80 | if kind == "record" { |
| 81 | sourceKind = "none" |
| 82 | } |
| 83 | if kind == "different stamp" { |
| 84 | sourceKind = "list" |
| 85 | } |
| 86 | f := r2Fixture(t, sourceKind) |
| 87 | |
| 88 | if kind == "record" { |
| 89 | f.rt.Account = "bab" |
| 90 | plan14PutRecord(t, f.e.store, provision.Record{Schema: 1, Host: f.rt.Host, Account: "bab", App: f.rt.App, Kind: f.rt.Kind, Unit: f.rt.Unit, Addr: f.rt.Addr, Channel: f.rt.Channel, Health: &provision.Health{Path: "/ready"}}) |
| 91 | } |
| 92 | u := &fix5Units{units: map[string]*r2Units{f.rt.Unit: f.u}} |
| 93 | target := f.u |
| 94 | if kind == "different stamp" { |
| 95 | copyUnit := *f.u.plan14RestartUnits |
| 96 | copyUnit.props.Name, copyUnit.props.MainPID = "stamped.service", 700 |
| 97 | target = &r2Units{plan14RestartUnits: ©Unit} |
| 98 | f.rt.Unit = copyUnit.props.Name |
| 99 | u.units[f.rt.Unit] = target |
| 100 | } |
| 101 | if err := f.e.root.reg.Upsert(f.rt); err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | f.source.sys = u |
| 105 | if kind != "none" && !(kind == "record" && os.Getenv("CARLOS_HOST_CLAIM") == "1") { |
| 106 | if _, err := f.source.forHost(t.Context(), f.rt.Host); err != nil { |
| 107 | t.Fatal("fixture must bind", err) |
| 108 | } |
| 109 | } |
| 110 | incident := strings.HasPrefix(history, "incident") || history == "TestR4ForwardResetFailedError" |
| 111 | if incident { |
| 112 | if err := release.StageArtifact(f.u.opt, "", "v2", f.rt.App, []byte("new build")); err != nil { |
| 113 | t.Fatal(err) |
| 114 | } |
| 115 | if err := release.PointLive(f.u.opt, f.rt.Host, "", "v2", f.rt.App); err != nil { |
| 116 | t.Fatal(err) |
| 117 | } |
| 118 | if err := f.e.root.reg.SetVersions(f.rt.Host, "v2", "v1"); err != nil { |
| 119 | t.Fatal(err) |
| 120 | } |
| 121 | target.exe = release.VersionPath(f.u.opt, "", "v2", f.rt.App) |
| 122 | target.props.ActiveState, target.props.SubState, target.props.MainPID = "failed", "failed", 0 |
| 123 | f.promote(t, "v1") |
| 124 | } |
| 125 | switch history { |
| 126 | case "crash", "failed-then-reset-failed before rollback": |
| 127 | target.crashNew = true |
| 128 | case "auto-restart": |
| 129 | target.props.ActiveState, target.props.SubState, target.props.MainPID = "activating", "auto-restart", 0 |
| 130 | case "inactive": |
| 131 | target.props.ActiveState, target.props.SubState, target.props.MainPID = "inactive", "dead", 0 |
| 132 | case "incident reset before": |
| 133 | if err := u.ResetFailed(f.rt.Unit); err != nil { |
| 134 | t.Fatal(err) |
| 135 | } |
| 136 | case "TestR4ForwardResetFailedError": |
| 137 | u.resetError = true |
| 138 | case "TestR4ForwardTryRestartError": |
| 139 | u.tryError = true |
| 140 | case "interrupted repoint": |
| 141 | // A root pass crashed after pointing v2 live and before restarting |
| 142 | // v1's process; the next build, v3, then fails to come up. Main |
| 143 | // rolls back to what the link names (v2), not to the running v1. |
| 144 | account := release.RouteAccount(f.rt) |
| 145 | if err := release.StageArtifact(f.u.opt, account, "v2", f.rt.App, []byte("new build")); err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | if err := release.PointLive(f.u.opt, f.rt.Host, account, "v2", f.rt.App); err != nil { |
| 149 | t.Fatal(err) |
| 150 | } |
| 151 | m := release.Manifest{App: f.rt.App, Version: "v3", Kind: "binary", Artifacts: map[string]release.Artifact{release.AdoptPlatform(): {}}} |
| 152 | if err := release.Ship(t.Context(), f.e.store, "bab", m, map[string][]byte{release.AdoptPlatform(): []byte("third build")}); err != nil { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | f.promote(t, "v3") |
| 156 | v3 := release.VersionPath(f.u.opt, account, "v3", f.rt.App) |
| 157 | target.afterEffect = func() { |
| 158 | if target.exe == v3 { |
| 159 | target.props.ActiveState, target.props.SubState, target.props.MainPID = "failed", "failed", 0 |
| 160 | } |
| 161 | } |
| 162 | } |
| 163 | getenv := func(string) string { return "5s" } |
| 164 | observer := f.e.root.obs |
| 165 | observer.resolve, observer.sys = f.c.resolve, u |
| 166 | observer.procExe = func(pid int) string { exe, _ := u.Exe(pid); return exe } |
| 167 | pr := f.e.root |
| 168 | pr.obs = observer |
| 169 | restart := rootRestartRouteWith(f.e.root.reg, u, f.u.opt, getenv, observer.recordSettled(f.e.bake), f.c) |
| 170 | rollback := rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, f.c) |
| 171 | startChecks := f.c |
| 172 | startChecks.legacyStart = u.Start |
| 173 | start := rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, startChecks) |
| 174 | if main { |
| 175 | restart = makeRestartRoute(f.e.root.reg, func(host string) { _ = u.TryRestart(activator.UnitName(host)) }, fix5MainCheckedRestart(u, f.u.opt, getenv, nil, f.c), nil) |
| 176 | rollback = fix5MainRollbackRestart(f.e.root.reg, u, f.u.opt, getenv, f.c) |
| 177 | start = makeStartRoute(f.e.root.reg, u.Start) |
| 178 | } else { |
| 179 | // Binding may observe health and retain a running build, but cannot enter |
| 180 | // the strict effect machinery or touch its durable restart journal. |
| 181 | f.c.attempts = func() (rootstate.Store, error) { |
| 182 | t.Fatal("no-census pass opened restart journal") |
| 183 | return nil, os.ErrPermission |
| 184 | } |
| 185 | f.c.dispatch = func(context.Context, registry.Route) (bool, error) { |
| 186 | t.Fatal("no-census pass dispatched binding") |
| 187 | return false, os.ErrPermission |
| 188 | } |
| 189 | restart = rootRestartRouteWith(f.e.root.reg, u, f.u.opt, getenv, observer.recordSettled(f.e.bake), f.c) |
| 190 | rollback = rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, f.c) |
| 191 | startChecks = f.c |
| 192 | startChecks.legacyStart = u.Start |
| 193 | start = rollbackUnitRestart(f.e.root.reg, u, f.u.opt, getenv, startChecks) |
| 194 | } |
| 195 | originalRollback := rollback |
| 196 | rollback = func(host string) error { |
| 197 | if history == "failed-then-reset-failed before rollback" { |
| 198 | if err := u.ResetFailed(f.rt.Unit); err != nil { |
| 199 | return err |
| 200 | } |
| 201 | } |
| 202 | return originalRollback(host) |
| 203 | } |
| 204 | snapshot := func() fix5Result { |
| 205 | t.Helper() |
| 206 | live, err := release.LiveVersion(f.u.opt, f.rt.Host) |
| 207 | if err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | previous, err := filepath.EvalSymlinks(filepath.Join(f.u.opt, "previous", f.rt.Host)) |
| 211 | if err != nil && !os.IsNotExist(err) { |
| 212 | t.Fatal(err) |
| 213 | } |
| 214 | rt, err := f.e.root.reg.Get(f.rt.Host) |
| 215 | if err != nil { |
| 216 | t.Fatal(err) |
| 217 | } |
| 218 | rt.CreatedAt = 0 |
| 219 | processes := map[string]string{} |
| 220 | for name, unit := range u.units { |
| 221 | processes[name] = strings.TrimPrefix(unit.exe, f.u.opt) + "|" + unit.props.ActiveState + "|" + unit.props.SubState + "|" + unit.props.InvocationID |
| 222 | } |
| 223 | return fix5Result{append([]string(nil), u.events...), live, strings.TrimPrefix(previous, f.u.opt), *rt, processes} |
| 224 | } |
| 225 | pass := func() { |
| 226 | t.Helper() |
| 227 | run := func() error { |
| 228 | opts := []release.PassOption{release.WithStartUnit(start), release.WithRollbackRestart(rollback)} |
| 229 | if !main { |
| 230 | recovery := productionNamedUnitRecovery(f.e.store, f.pub, f.u.opt, hostwrite.Config{}) |
| 231 | opts = append(opts, release.WithRootHistory(nil), release.WithRunningVersion(f.c.runningVersion), release.WithRestartAttemptSettled(f.c.finish), |
| 232 | recovery.passOption(f.e.root.reg, f.c, getenv)) |
| 233 | } |
| 234 | return release.Pass(t.Context(), f.e.root.reg, f.e.store, f.pub, f.u.opt, restart, opts...) |
| 235 | } |
| 236 | var err error |
| 237 | if main { |
| 238 | err = run() |
| 239 | } else { |
| 240 | err = pr.around(t.Context(), run) |
| 241 | } |
| 242 | if err != nil { |
| 243 | t.Fatal("pass status differs from main", err) |
| 244 | } |
| 245 | } |
| 246 | pass() |
| 247 | results := []fix5Result{snapshot()} |
| 248 | if incident { |
| 249 | u.resetError = false |
| 250 | if history != "incident reset before" { |
| 251 | if err := u.ResetFailed(f.rt.Unit); err != nil { |
| 252 | t.Fatal(err) |
| 253 | } |
| 254 | } |
| 255 | if err := u.Start(f.rt.Unit); err != nil { |
| 256 | t.Fatal(err) |
| 257 | } |
| 258 | results = append(results, snapshot()) |
| 259 | } |
| 260 | // A second pass detects a synthetic refusal that would reject the fix. |
| 261 | pass() |
| 262 | return append(results, snapshot()) |
| 263 | } |
| 264 | |
| 265 | func TestPlan14Fix5MainEquivalence(t *testing.T) { |
| 266 | slots := make(chan struct{}, 4) |
| 267 | if !plan14RootFixture(t) { |
| 268 | return |
| 269 | } |
| 270 | for _, flag := range []string{"", "1"} { |
| 271 | t.Run(map[string]string{"": "claims-off", "1": "claims-on"}[flag], func(t *testing.T) { |
| 272 | t.Setenv("CARLOS_HOST_CLAIM", flag) |
| 273 | for _, kind := range []string{"none", "record", "list", "guard", "different stamp"} { |
| 274 | for _, history := range []string{"healthy", "crash", "failed-then-reset-failed before rollback", "auto-restart", "inactive", "incident reset before", "incident reset after", "TestR4ForwardResetFailedError", "TestR4ForwardTryRestartError", "interrupted repoint"} { |
| 275 | t.Run(kind+"/"+history, func(t *testing.T) { |
| 276 | t.Parallel() |
| 277 | slots <- struct{}{} |
| 278 | defer func() { <-slots }() |
| 279 | want := fix5Run(t, kind, history, true) |
| 280 | got := fix5Run(t, kind, history, false) |
| 281 | if !reflect.DeepEqual(got, want) { |
| 282 | t.Fatalf("no-census effects/pointers differ from main:\ngot %+v\nwant %+v", got, want) |
| 283 | } |
| 284 | }) |
| 285 | } |
| 286 | } |
| 287 | }) |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | func TestPlan14Fix5NoCensusSkipsJournalRecovery(t *testing.T) { |
| 292 | if !plan14RootFixture(t) { |
| 293 | return |
| 294 | } |
| 295 | for _, flag := range []string{"", "1"} { |
| 296 | t.Setenv("CARLOS_HOST_CLAIM", flag) |
| 297 | f := r2Fixture(t, "list") |
| 298 | f.c.attempts = func() (rootstate.Store, error) { |
| 299 | t.Fatal("read retained journal without a census") |
| 300 | return nil, os.ErrPermission |
| 301 | } |
| 302 | if err := recoverRootRestarts(t.Context(), f.u, f.u.opt, nil, f.c, hostwrite.Config{}); err != nil { |
| 303 | t.Fatal(err) |
| 304 | } |
| 305 | if len(f.u.effects) != 0 { |
| 306 | t.Fatal(f.u.effects) |
| 307 | } |
| 308 | } |
| 309 | } |
| 310 | |
amadan