| 1 | package host |
| 2 | |
| 3 | // The box half of a narrowed restart (discussion #16): that a request naming |
| 4 | // one route cycles one route, that the rules restart.go pays for — seed |
| 5 | // don't replay, never wake a sleeper, one cycle per host per pass — survive |
| 6 | // the narrowing, and that a selector this build cannot read cycles NOTHING. |
| 7 | |
| 8 | import ( |
| 9 | "bytes" |
| 10 | "context" |
| 11 | "crypto/ed25519" |
| 12 | "encoding/json" |
| 13 | "errors" |
| 14 | "fmt" |
| 15 | "log" |
| 16 | "os" |
| 17 | "path/filepath" |
| 18 | "slices" |
| 19 | "strings" |
| 20 | "testing" |
| 21 | "time" |
| 22 | |
| 23 | "github.com/carlosframework/platform/internal/registry" |
| 24 | "github.com/carlosframework/platform/internal/release" |
| 25 | "github.com/carlosframework/platform/internal/rollout" |
| 26 | ) |
| 27 | |
| 28 | // askScoped is a member's `carlos restart --host/--environment` landing in the |
| 29 | // bucket. |
| 30 | func askScoped(t *testing.T, ctx context.Context, s release.Store, acct, app, kind, value string) release.RestartRequest { |
| 31 | t.Helper() |
| 32 | req, err := release.AppendRestartScope(ctx, s, acct, app, release.RestartScope{Kind: kind, Value: value}) |
| 33 | if err != nil { |
| 34 | t.Fatalf("AppendRestartScope %s/%s (%s %s): %v", acct, app, kind, value, err) |
| 35 | } |
| 36 | return req |
| 37 | } |
| 38 | |
| 39 | // envRoute is a unit-backed instance bound to a named config bundle — the |
| 40 | // shape `--environment` exists for, and the shape only the root pass can |
| 41 | // cycle. |
| 42 | func envRoute(host, acct, app, env string) registry.Route { |
| 43 | rt := unitRoute(host, acct, app) |
| 44 | rt.Environment = env |
| 45 | return rt |
| 46 | } |
| 47 | |
| 48 | // titogoRoutes is the fixture the whole discussion is about, small: one app, |
| 49 | // several routes, two bundles between them, one on nothing. |
| 50 | func titogoRoutes() []registry.Route { |
| 51 | return []registry.Route{ |
| 52 | envRoute("connect.example.com", "bab", "titogo", "relay"), |
| 53 | envRoute("relay2.example.com", "bab", "titogo", "relay"), |
| 54 | envRoute("platform.example.com", "bab", "titogo", "router"), |
| 55 | unitRoute("bill.example.com", "bab", "titogo"), |
| 56 | } |
| 57 | } |
| 58 | |
| 59 | // ---- the agent's sweep ------------------------------------------------------- |
| 60 | |
| 61 | // TestRestartScopePassFirstSightIsSilent: a box that has never looked records |
| 62 | // where the log stands and asks for nothing. Without this, a box coming up |
| 63 | // after a request would cycle for it a second time — and a box whose memory |
| 64 | // was lost would cycle for every request still in the log. |
| 65 | func TestRestartScopePassFirstSightIsSilent(t *testing.T) { |
| 66 | ctx, s := genKit(t) |
| 67 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com") |
| 68 | cache := map[[2]string]string{} |
| 69 | work, errs := restartScopePass(ctx, titogoRoutes(), s, cache) |
| 70 | if len(work) != 0 || len(errs) != 0 { |
| 71 | t.Fatalf("first sight asked for %v (errs %v) — it must only record", work, errs) |
| 72 | } |
| 73 | if len(cache) != 1 { |
| 74 | t.Fatalf("cache = %v, want the app recorded", cache) |
| 75 | } |
| 76 | } |
| 77 | |
| 78 | // TestRestartScopePassCarriesEveryRequestInAWindow is why the object is a log: |
| 79 | // two requests between two sweeps are TWO selectors, and collapsing them into |
| 80 | // the last one would silently drop a restart somebody was told was requested. |
| 81 | func TestRestartScopePassCarriesEveryRequestInAWindow(t *testing.T) { |
| 82 | ctx, s := genKit(t) |
| 83 | routes := titogoRoutes() |
| 84 | cache := map[[2]string]string{} |
| 85 | restartScopePass(ctx, routes, s, cache) // baseline |
| 86 | |
| 87 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com") |
| 88 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "router") |
| 89 | work, _ := restartScopePass(ctx, routes, s, cache) |
| 90 | if len(work) != 1 || len(work[0].requests) != 2 { |
| 91 | t.Fatalf("work = %+v, want one app carrying both requests", work) |
| 92 | } |
| 93 | hosts := scopedHosts(routes, work[0].app, work[0].requests, func(registry.Route) bool { return true }) |
| 94 | if !slices.Equal(hosts, []string{"bill.example.com", "platform.example.com"}) { |
| 95 | t.Fatalf("addressed %v, want exactly the two named routes", hosts) |
| 96 | } |
| 97 | // And an unmoved log asks for nothing on the next sweep. |
| 98 | if work, _ := restartScopePass(ctx, routes, s, cache); len(work) != 0 { |
| 99 | t.Fatalf("an unmoved log asked for %+v", work) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | // TestRestartScopePassReBaselinesALostPosition is the agent's own half of the |
| 104 | // rolled-off-mark rule, exercised directly rather than through the root pass: |
| 105 | // a box away long enough for its position to leave the log has no honest |
| 106 | // answer to "what did I miss", so it records where the log now stands and asks |
| 107 | // for nothing. Replaying instead would cycle routes for requests served days |
| 108 | // ago; a dropped one is recovered by typing the verb again. |
| 109 | func TestRestartScopePassReBaselinesALostPosition(t *testing.T) { |
| 110 | ctx, s := genKit(t) |
| 111 | routes := titogoRoutes() |
| 112 | req := askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "relay") |
| 113 | |
| 114 | // This box acted up to a request that has since been trimmed off the front |
| 115 | // of the log. |
| 116 | cache := map[[2]string]string{{"bab", "titogo"}: "rolled-off"} |
| 117 | work, errs := restartScopePass(ctx, routes, s, cache) |
| 118 | if len(errs) != 0 { |
| 119 | t.Fatalf("unexpected read errors: %v", errs) |
| 120 | } |
| 121 | if len(work) != 0 { |
| 122 | t.Fatalf("a lost position asked for %+v — it must re-baseline instead of replaying the log", work) |
| 123 | } |
| 124 | if got := cache[[2]string{"bab", "titogo"}]; got != req.ID { |
| 125 | t.Fatalf("cache = %q, want the re-baseline to the log's newest %q", got, req.ID) |
| 126 | } |
| 127 | // And the NEXT request is acted on: re-baselining is not a box giving up. |
| 128 | next := askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com") |
| 129 | work, _ = restartScopePass(ctx, routes, s, cache) |
| 130 | if len(work) != 1 || len(work[0].requests) != 1 || work[0].requests[0].ID != next.ID { |
| 131 | t.Fatalf("work = %+v, want the one request made after the re-baseline", work) |
| 132 | } |
| 133 | } |
| 134 | |
| 135 | // TestScopedHostsNarrowsToTheSelector is the whole complaint, tested: one |
| 136 | // credential delivered to one bundle must cycle the routes reading that |
| 137 | // bundle, not every route the app owns. |
| 138 | func TestScopedHostsNarrowsToTheSelector(t *testing.T) { |
| 139 | routes := titogoRoutes() |
| 140 | app := [2]string{"bab", "titogo"} |
| 141 | all := func(registry.Route) bool { return true } |
| 142 | |
| 143 | got := scopedHosts(routes, app, []release.RestartRequest{ |
| 144 | {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeEnvironment, Value: "relay"}}, |
| 145 | }, all) |
| 146 | if !slices.Equal(got, []string{"connect.example.com", "relay2.example.com"}) { |
| 147 | t.Fatalf("environment relay addressed %v, want the two relay routes and nothing else", got) |
| 148 | } |
| 149 | got = scopedHosts(routes, app, []release.RestartRequest{ |
| 150 | {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "bill.example.com"}}, |
| 151 | }, all) |
| 152 | if !slices.Equal(got, []string{"bill.example.com"}) { |
| 153 | t.Fatalf("host selector addressed %v, want the one route", got) |
| 154 | } |
| 155 | // Another account's route with the same host is not this app's route. |
| 156 | other := append(routes, envRoute("connect.example.com", "zzz", "titogo", "relay")) |
| 157 | got = scopedHosts(other, app, []release.RestartRequest{ |
| 158 | {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "connect.example.com"}}, |
| 159 | }, all) |
| 160 | if !slices.Equal(got, []string{"connect.example.com"}) { |
| 161 | t.Fatalf("addressed %v — a host must be resolved within this account and app only", got) |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | // TestScopedHostsIgnoresAnUnknownSelector: the forward-compatibility rule at |
| 166 | // the point it matters. A `--region` invented after this binary shipped must |
| 167 | // address nothing here, never fall back to the app. |
| 168 | func TestScopedHostsIgnoresAnUnknownSelector(t *testing.T) { |
| 169 | got := scopedHosts(titogoRoutes(), [2]string{"bab", "titogo"}, []release.RestartRequest{ |
| 170 | {ID: "1", Scope: release.RestartScope{Kind: "region", Value: "eu-west-1"}}, |
| 171 | }, func(registry.Route) bool { return true }) |
| 172 | if len(got) != 0 { |
| 173 | t.Fatalf("an unreadable selector addressed %v — a box must never widen what it cannot understand", got) |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | // TestScopedHostsDeduplicates: two requests naming the same route mean one |
| 178 | // cycle of it. A verb whose job is stopping processes must not manufacture a |
| 179 | // second stop out of somebody typing the same thing twice. |
| 180 | func TestScopedHostsDeduplicates(t *testing.T) { |
| 181 | got := scopedHosts(titogoRoutes(), [2]string{"bab", "titogo"}, []release.RestartRequest{ |
| 182 | {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "connect.example.com"}}, |
| 183 | {ID: "2", Scope: release.RestartScope{Kind: release.RestartScopeEnvironment, Value: "relay"}}, |
| 184 | }, func(registry.Route) bool { return true }) |
| 185 | if !slices.Equal(got, []string{"connect.example.com", "relay2.example.com"}) { |
| 186 | t.Fatalf("addressed %v, want each route once", got) |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | // TestBounceScopedRoutesCyclesOnlyItsOwnChildren: the agent's half reaches |
| 191 | // exec children and nothing else, exactly as the app-wide bounce does — the |
| 192 | // two halves must never cycle one process twice. |
| 193 | func TestBounceScopedRoutesCyclesOnlyItsOwnChildren(t *testing.T) { |
| 194 | routes := []registry.Route{ |
| 195 | execRoute("exec.example.com", "bab", "titogo"), |
| 196 | unitRoute("unit.example.com", "bab", "titogo"), |
| 197 | } |
| 198 | routes[0].Environment, routes[1].Environment = "relay", "relay" |
| 199 | var bounced []string |
| 200 | n := bounceScopedRoutes(routes, []scopedRestart{{ |
| 201 | app: [2]string{"bab", "titogo"}, |
| 202 | requests: []release.RestartRequest{{ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeEnvironment, Value: "relay"}}}, |
| 203 | }}, func(h string) bool { bounced = append(bounced, h); return true }) |
| 204 | if n != 1 || !slices.Equal(bounced, []string{"exec.example.com"}) { |
| 205 | t.Fatalf("bounced %v (n=%d), want the exec child only — the unit route is root's", bounced, n) |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | // TestRestartRequestLegCarriesBothSignals: an app-wide request and a narrowed |
| 210 | // one can land in the same poll window, and the leg that stopped at the first |
| 211 | // would leave the other to the hourly floor. |
| 212 | func TestRestartRequestLegCarriesBothSignals(t *testing.T) { |
| 213 | ctx, s := genKit(t) |
| 214 | dd := t.TempDir() |
| 215 | routes := []registry.Route{ |
| 216 | execRoute("a.example.com", "bab", "eleven"), |
| 217 | execRoute("b.example.com", "bab", "eleven"), |
| 218 | } |
| 219 | routes[1].Environment = "relay" |
| 220 | mem := newRestartMemory() |
| 221 | var bounced []string |
| 222 | bounce := func(h string) bool { bounced = append(bounced, h); return true } |
| 223 | if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked { |
| 224 | t.Fatal("the baselining leg asked for a root pass") |
| 225 | } |
| 226 | askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeEnvironment, "relay") |
| 227 | if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); !asked { |
| 228 | t.Fatal("a narrowed request did not ask for a root pass") |
| 229 | } |
| 230 | if !slices.Equal(bounced, []string{"b.example.com"}) { |
| 231 | t.Fatalf("bounced %v, want only the route reading relay", bounced) |
| 232 | } |
| 233 | if _, err := os.Stat(filepath.Join(dd, restartRequestMarker)); err != nil { |
| 234 | t.Fatalf("no marker after a narrowed request: %v", err) |
| 235 | } |
| 236 | } |
| 237 | |
| 238 | // ---- the root pass ----------------------------------------------------------- |
| 239 | |
| 240 | // TestRestartPassCyclesOnlyTheNarrowedRoutes is the fix itself: a request |
| 241 | // naming one environment cycles the routes on it, and leaves the other |
| 242 | // seventeen production routes running. |
| 243 | func TestRestartPassCyclesOnlyTheNarrowedRoutes(t *testing.T) { |
| 244 | ctx, s := genKit(t) |
| 245 | dd := t.TempDir() |
| 246 | routes := titogoRoutes() |
| 247 | marks := restartMarksPath(dd) |
| 248 | |
| 249 | restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn) // baseline |
| 250 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "relay") |
| 251 | rec := &recordingRestart{} |
| 252 | restartPass(ctx, routes, s, marks, rec.fn) |
| 253 | if !slices.Equal(rec.hosts, []string{"connect.example.com", "relay2.example.com"}) { |
| 254 | t.Fatalf("cycled %v, want only the routes reading relay", rec.hosts) |
| 255 | } |
| 256 | // And the request is not served twice. |
| 257 | rec2 := &recordingRestart{} |
| 258 | restartPass(ctx, routes, s, marks, rec2.fn) |
| 259 | if len(rec2.hosts) != 0 { |
| 260 | t.Fatalf("a second pass cycled %v for a request already served", rec2.hosts) |
| 261 | } |
| 262 | } |
| 263 | |
| 264 | // TestRestartPassSeedsNarrowedRequestsWithoutReplaying: a box that has never |
| 265 | // looked at the log records where it stands and cycles nothing — the property |
| 266 | // that keeps a lost marks file from bouncing a box's whole route table. |
| 267 | func TestRestartPassSeedsNarrowedRequestsWithoutReplaying(t *testing.T) { |
| 268 | ctx, s := genKit(t) |
| 269 | dd := t.TempDir() |
| 270 | routes := titogoRoutes() |
| 271 | req := askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com") |
| 272 | |
| 273 | rec := &recordingRestart{} |
| 274 | restartPass(ctx, routes, s, restartMarksPath(dd), rec.fn) |
| 275 | if len(rec.hosts) != 0 { |
| 276 | t.Fatalf("the seeding pass cycled %v", rec.hosts) |
| 277 | } |
| 278 | m := marksOf(t, restartMarksPath(dd)) |
| 279 | if got := m.Scopes[restartMarkKey("bab", "titogo")]; got != req.ID { |
| 280 | t.Fatalf("scope mark = %q, want the newest request %q", got, req.ID) |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | // TestRestartPassLeavesAHibernatingRouteAsleep: the load-bearing one. A |
| 285 | // narrowed request that names a sleeping instance must not wake it — the |
| 286 | // instance execs fresh on its next wake, which IS the restart. |
| 287 | func TestRestartPassLeavesAHibernatingRouteAsleep(t *testing.T) { |
| 288 | ctx, s := genKit(t) |
| 289 | dd := t.TempDir() |
| 290 | sleeper := envRoute("sleeper.example.com", "bab", "titogo", "relay") |
| 291 | sleeper.Hibernate = true |
| 292 | routes := append(titogoRoutes(), sleeper) |
| 293 | marks := restartMarksPath(dd) |
| 294 | |
| 295 | restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn) |
| 296 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "relay") |
| 297 | rec := &recordingRestart{} |
| 298 | restartPass(ctx, routes, s, marks, rec.fn) |
| 299 | if slices.Contains(rec.hosts, "sleeper.example.com") { |
| 300 | t.Fatalf("cycled %v — a hibernating instance was woken by a restart request", rec.hosts) |
| 301 | } |
| 302 | if !slices.Equal(rec.hosts, []string{"connect.example.com", "relay2.example.com"}) { |
| 303 | t.Fatalf("cycled %v, want the two awake relay routes", rec.hosts) |
| 304 | } |
| 305 | } |
| 306 | |
| 307 | // TestRestartPassCyclesEachHostOnceAcrossBothSignals: an app-wide request and |
| 308 | // a narrowed one landing in the same window are one cycle per route, not two. |
| 309 | func TestRestartPassCyclesEachHostOnceAcrossBothSignals(t *testing.T) { |
| 310 | ctx, s := genKit(t) |
| 311 | dd := t.TempDir() |
| 312 | routes := titogoRoutes() |
| 313 | marks := restartMarksPath(dd) |
| 314 | |
| 315 | restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn) |
| 316 | askRestart(t, ctx, s, "bab", "titogo") |
| 317 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "relay") |
| 318 | rec := &recordingRestart{} |
| 319 | restartPass(ctx, routes, s, marks, rec.fn) |
| 320 | slices.Sort(rec.hosts) |
| 321 | want := []string{"bill.example.com", "connect.example.com", "platform.example.com", "relay2.example.com"} |
| 322 | if !slices.Equal(rec.hosts, want) { |
| 323 | t.Fatalf("cycled %v, want each route exactly once: %v", rec.hosts, want) |
| 324 | } |
| 325 | } |
| 326 | |
| 327 | // TestRestartPassIgnoresAnUnreadableSelector: the box-level half of the |
| 328 | // never-widen rule. A request this build cannot interpret cycles nothing, and |
| 329 | // the mark still advances so the pass does not re-read it hourly forever. |
| 330 | func TestRestartPassIgnoresAnUnreadableSelector(t *testing.T) { |
| 331 | ctx, s := genKit(t) |
| 332 | dd := t.TempDir() |
| 333 | routes := titogoRoutes() |
| 334 | marks := restartMarksPath(dd) |
| 335 | restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn) |
| 336 | |
| 337 | // Written straight to the bucket, because AppendRestartScope refuses to |
| 338 | // write a kind it does not know — this is a NEWER console's request |
| 339 | // reaching an older box, which is the case that matters. |
| 340 | raw, err := json.Marshal(release.ScopedRestartGeneration{Schema: 1, Requests: []release.RestartRequest{ |
| 341 | {ID: "future-1", Scope: release.RestartScope{Kind: "region", Value: "eu-west-1"}}, |
| 342 | }}) |
| 343 | if err != nil { |
| 344 | t.Fatalf("marshal: %v", err) |
| 345 | } |
| 346 | if _, err := s.Put(ctx, release.RestartScopeKey("bab", "titogo"), raw, ""); err != nil { |
| 347 | t.Fatalf("put a future request: %v", err) |
| 348 | } |
| 349 | rec := &recordingRestart{} |
| 350 | restartPass(ctx, routes, s, marks, rec.fn) |
| 351 | if len(rec.hosts) != 0 { |
| 352 | t.Fatalf("cycled %v for a selector this build cannot read — it must cycle nothing", rec.hosts) |
| 353 | } |
| 354 | if got := marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")]; got != "future-1" { |
| 355 | t.Fatalf("scope mark = %q, want the request stepped over rather than re-read forever", got) |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | // TestRestartPassReBaselinesALostPosition: a box away long enough for its mark |
| 360 | // to roll off the log re-baselines and cycles nothing. A dropped request is |
| 361 | // recoverable by typing the verb again; a replayed one is not. |
| 362 | func TestRestartPassReBaselinesALostPosition(t *testing.T) { |
| 363 | ctx, s := genKit(t) |
| 364 | dd := t.TempDir() |
| 365 | routes := titogoRoutes() |
| 366 | marks := restartMarksPath(dd) |
| 367 | restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn) |
| 368 | |
| 369 | raw, err := json.Marshal(release.ScopedRestartGeneration{Schema: 1, Requests: []release.RestartRequest{ |
| 370 | {ID: "gone", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "bill.example.com"}}, |
| 371 | }}) |
| 372 | if err != nil { |
| 373 | t.Fatalf("marshal: %v", err) |
| 374 | } |
| 375 | if _, err := s.Put(ctx, release.RestartScopeKey("bab", "titogo"), raw, ""); err != nil { |
| 376 | t.Fatalf("seed a log: %v", err) |
| 377 | } |
| 378 | // This box acted up to a request that has since been trimmed away. |
| 379 | m := marksOf(t, marks) |
| 380 | m.Scopes[restartMarkKey("bab", "titogo")] = "rolled-off" |
| 381 | saveMarks(t, marks, m) |
| 382 | |
| 383 | rec := &recordingRestart{} |
| 384 | restartPass(ctx, routes, s, marks, rec.fn) |
| 385 | if len(rec.hosts) != 0 { |
| 386 | t.Fatalf("cycled %v after losing its position — it must re-baseline instead", rec.hosts) |
| 387 | } |
| 388 | if got := marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")]; got != "gone" { |
| 389 | t.Fatalf("scope mark = %q, want the re-baseline to the log's newest", got) |
| 390 | } |
| 391 | } |
| 392 | |
| 393 | // ---- the handover on a box that predates narrowing --------------------------- |
| 394 | |
| 395 | // writePreNarrowingMarks puts on disk exactly what the binary before this |
| 396 | // feature wrote: schema, apps, and no `scopes` key at all. Every box that has |
| 397 | // ever run the app-wide restart has one of these. |
| 398 | func writePreNarrowingMarks(t *testing.T, path string, apps map[string]string) { |
| 399 | t.Helper() |
| 400 | raw, err := json.Marshal(map[string]any{"schema": 1, "apps": apps}) |
| 401 | if err != nil { |
| 402 | t.Fatalf("marshal old marks: %v", err) |
| 403 | } |
| 404 | if err := os.WriteFile(path, raw, 0o644); err != nil { |
| 405 | t.Fatalf("write old marks: %v", err) |
| 406 | } |
| 407 | } |
| 408 | |
| 409 | // armAgent is one agent start: the two silent sweeps that fill its memory, and |
| 410 | // the seed that either writes the root pass's baseline or reports that the one |
| 411 | // already on disk is missing a narrowed mark. |
| 412 | func armAgent(t *testing.T, ctx context.Context, routes []registry.Route, s release.Store, dd string) (*restartMemory, bool) { |
| 413 | t.Helper() |
| 414 | mem := newRestartMemory() |
| 415 | if _, errs := restartGenPass(ctx, routes, s, mem.gen); len(errs) != 0 { |
| 416 | t.Fatalf("gen sweep: %v", errs) |
| 417 | } |
| 418 | if _, errs := restartScopePass(ctx, routes, s, mem.scope); len(errs) != 0 { |
| 419 | t.Fatalf("scope sweep: %v", errs) |
| 420 | } |
| 421 | need, err := seedRestartMarks(restartMarksPath(dd), mem) |
| 422 | if err != nil { |
| 423 | t.Fatalf("seed: %v", err) |
| 424 | } |
| 425 | return mem, need |
| 426 | } |
| 427 | |
| 428 | // TestAPreNarrowingMarksFileIsCompletedBeforeTheNextRequestIsDropped is the |
| 429 | // shape of every production box on the day this rolls, and the drop it used to |
| 430 | // cause. |
| 431 | // |
| 432 | // The marks file was written before narrowed restarts existed, so it has |
| 433 | // `apps` and no `scopes`. The agent's create-only seed will not touch it, |
| 434 | // which leaves the ROOT pass with no narrowed position — and a missing |
| 435 | // position reads as a FIRST SIGHT, so that pass records the log's newest id |
| 436 | // and cycles nothing. An operator who ran `carlos restart --host relay2` in |
| 437 | // that window was told "restart requested" and got no restart: the exec-shaped |
| 438 | // routes survive because the agent bounces its own children out of memory, and |
| 439 | // the unit-shaped ones — the relay and the router these flags exist for — are |
| 440 | // the ones that vanish. |
| 441 | // |
| 442 | // The fix is the seed REPORTING the gap so that one root pass records the |
| 443 | // marks. This walks the whole handover: arm, let that pass run, then ask. |
| 444 | func TestAPreNarrowingMarksFileIsCompletedBeforeTheNextRequestIsDropped(t *testing.T) { |
| 445 | ctx, s := genKit(t) |
| 446 | dd := t.TempDir() |
| 447 | routes := titogoRoutes() // unit-shaped: only the root pass can cycle these |
| 448 | marks := restartMarksPath(dd) |
| 449 | // A request from before the roll, already in the log. Nothing here may |
| 450 | // replay it. |
| 451 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "router") |
| 452 | writePreNarrowingMarks(t, marks, map[string]string{ |
| 453 | restartMarkKey("bab", "titogo"): genAbsent, // no app-wide object; the old binary recorded the absence |
| 454 | }) |
| 455 | |
| 456 | if _, need := armAgent(t, ctx, routes, s, dd); !need { |
| 457 | t.Fatal("the agent saw a marks file with no narrowed mark and asked for nothing — " + |
| 458 | `the next narrowed request would be answered "requested" and cycle nothing unit-shaped`) |
| 459 | } |
| 460 | if got := marksOf(t, marks).Scopes; len(got) != 0 { |
| 461 | t.Fatalf("the seed wrote %v into an existing marks file — it must only report, never race the root pass's own write", got) |
| 462 | } |
| 463 | |
| 464 | // The root pass that touch asks for. It records and cycles nothing: the |
| 465 | // request already in the log is this box's history, not work it owes. |
| 466 | rec := &recordingRestart{} |
| 467 | restartPass(ctx, routes, s, marks, rec.fn) |
| 468 | if len(rec.hosts) != 0 { |
| 469 | t.Fatalf("the baselining pass cycled %v — a box completing its baseline must replay nothing", rec.hosts) |
| 470 | } |
| 471 | if marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")] == "" { |
| 472 | t.Fatal("the pass recorded no scope mark, so the box would re-seed forever") |
| 473 | } |
| 474 | |
| 475 | // And NOW the operator types the verb. This is the request that used to be |
| 476 | // dropped. |
| 477 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "relay2.example.com") |
| 478 | rec = &recordingRestart{} |
| 479 | restartPass(ctx, routes, s, marks, rec.fn) |
| 480 | if !slices.Equal(rec.hosts, []string{"relay2.example.com"}) { |
| 481 | t.Fatalf("cycled %v, want the one route the operator named", rec.hosts) |
| 482 | } |
| 483 | |
| 484 | // A second agent start over a now-complete baseline asks for nothing: the |
| 485 | // repair is one pass per box, not one per agent start. |
| 486 | if _, need := armAgent(t, ctx, routes, s, dd); need { |
| 487 | t.Fatal("an agent restarting over a complete baseline asked for another root pass") |
| 488 | } |
| 489 | } |
| 490 | |
| 491 | // TestSeedOnAFreshBoxAsksForNoRootPass is the genuinely-first-sight case, kept |
| 492 | // separate because it is the one the create-only seed already handles: with no |
| 493 | // marks file at all the agent writes BOTH memories itself, so there is nothing |
| 494 | // left for a root pass to fill in — and the next narrowed request is acted on. |
| 495 | func TestSeedOnAFreshBoxAsksForNoRootPass(t *testing.T) { |
| 496 | ctx, s := genKit(t) |
| 497 | dd := t.TempDir() |
| 498 | routes := titogoRoutes() |
| 499 | marks := restartMarksPath(dd) |
| 500 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeEnvironment, "router") // history |
| 501 | |
| 502 | if _, need := armAgent(t, ctx, routes, s, dd); need { |
| 503 | t.Fatal("a seed that wrote the whole baseline itself still asked for a root pass") |
| 504 | } |
| 505 | if marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")] == "" { |
| 506 | t.Fatal("the fresh seed recorded no scope mark") |
| 507 | } |
| 508 | |
| 509 | askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com") |
| 510 | rec := &recordingRestart{} |
| 511 | restartPass(ctx, routes, s, marks, rec.fn) |
| 512 | if !slices.Equal(rec.hosts, []string{"bill.example.com"}) { |
| 513 | t.Fatalf("cycled %v, want the named route — the seeded baseline is what the root pass trusts", rec.hosts) |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | // TestRestartSeedAsksTheRootPassThroughTheMarker is the same repair through |
| 518 | // the real entry point, so the marker file — the only way this unprivileged |
| 519 | // process can reach a root pass at all — is pinned rather than assumed. |
| 520 | func TestRestartSeedAsksTheRootPassThroughTheMarker(t *testing.T) { |
| 521 | ctx, s := genKit(t) |
| 522 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 523 | if err != nil { |
| 524 | t.Fatal(err) |
| 525 | } |
| 526 | defer reg.Close() |
| 527 | for _, rt := range titogoRoutes() { |
| 528 | if err := reg.Upsert(rt); err != nil { |
| 529 | t.Fatal(err) |
| 530 | } |
| 531 | } |
| 532 | |
| 533 | dd := t.TempDir() |
| 534 | writePreNarrowingMarks(t, restartMarksPath(dd), map[string]string{ |
| 535 | restartMarkKey("bab", "titogo"): genAbsent, |
| 536 | }) |
| 537 | restartSeed(ctx, reg, s, dd, newRestartMemory()) |
| 538 | if _, err := os.Stat(filepath.Join(dd, restartRequestMarker)); err != nil { |
| 539 | t.Fatalf("no marker after arming over a pre-narrowing baseline: %v — "+ |
| 540 | "the root pass would record no narrowed mark until the hourly floor", err) |
| 541 | } |
| 542 | |
| 543 | // A baseline this agent wrote itself leaves the marker alone: an agent that |
| 544 | // touched it on every start would start a root pass on every start. |
| 545 | fresh := t.TempDir() |
| 546 | restartSeed(ctx, reg, s, fresh, newRestartMemory()) |
| 547 | if _, err := os.Stat(filepath.Join(fresh, restartRequestMarker)); !errors.Is(err, os.ErrNotExist) { |
| 548 | t.Fatalf("the fresh seed touched the marker (err %v), want no request at all", err) |
| 549 | } |
| 550 | } |
| 551 | |
| 552 | // TestRestartPassAdvancesTheScopeMarkOnAFailedDispatch: a request is a |
| 553 | // point-in-time event, not a desired state. A unit that will not come back |
| 554 | // must not be retried on every pass forever for something somebody asked once. |
| 555 | func TestRestartPassAdvancesTheScopeMarkOnAFailedDispatch(t *testing.T) { |
| 556 | ctx, s := genKit(t) |
| 557 | dd := t.TempDir() |
| 558 | routes := titogoRoutes() |
| 559 | marks := restartMarksPath(dd) |
| 560 | restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn) |
| 561 | |
| 562 | req := askScoped(t, ctx, s, "bab", "titogo", release.RestartScopeHost, "bill.example.com") |
| 563 | rec := &recordingRestart{fail: map[string]error{"bill.example.com": os.ErrPermission}} |
| 564 | restartPass(ctx, routes, s, marks, rec.fn) |
| 565 | if !slices.Equal(rec.hosts, []string{"bill.example.com"}) { |
| 566 | t.Fatalf("tried %v, want the named route", rec.hosts) |
| 567 | } |
| 568 | if got := marksOf(t, marks).Scopes[restartMarkKey("bab", "titogo")]; got != req.ID { |
| 569 | t.Fatalf("scope mark = %q after a failed dispatch, want %q — an unadvanced mark retries forever", got, req.ID) |
| 570 | } |
| 571 | } |
| 572 | |
| 573 | // saveMarks rewrites the marks file the way the pass does, so a test can put |
| 574 | // this box's memory where it needs it. |
| 575 | func saveMarks(t *testing.T, path string, m *restartMarks) { |
| 576 | t.Helper() |
| 577 | raw, err := json.Marshal(m) |
| 578 | if err != nil { |
| 579 | t.Fatalf("marshal marks: %v", err) |
| 580 | } |
| 581 | if err := os.WriteFile(path, raw, 0o644); err != nil { |
| 582 | t.Fatalf("write marks: %v", err) |
| 583 | } |
| 584 | } |
| 585 | |
| 586 | // A retry request clears one host's hold through the rollout manager, cycles |
| 587 | // nothing, and asks root for no restart pass. The grant's build, book and |
| 588 | // epoch are what the manager is handed, once. |
| 589 | func TestRestartLegHandsARetryToTheRolloutManager(t *testing.T) { |
| 590 | ctx, s := genKit(t) |
| 591 | dd := t.TempDir() |
| 592 | routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")} |
| 593 | mem, priv := retryMemory(t) |
| 594 | var retried []string |
| 595 | mem.retryHost = func(h, b, bk string, e uint64) bool { |
| 596 | retried = append(retried, fmt.Sprintf("%s %s %s %d", h, b, bk, e)) |
| 597 | return true |
| 598 | } |
| 599 | bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true } |
| 600 | if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked { |
| 601 | t.Fatal("the baselining leg asked for a root pass") |
| 602 | } |
| 603 | askRetry(t, ctx, s, priv, "bab", "eleven", "b.example.com", "v2", "bk", 5) |
| 604 | askRetry(t, ctx, s, priv, "bab", "eleven", "elsewhere.example.com", "v2", "bk", 5) |
| 605 | if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked { |
| 606 | t.Fatal("a retry-only move asked for a root restart pass") |
| 607 | } |
| 608 | if !slices.Equal(retried, []string{"b.example.com v2 bk 5"}) { |
| 609 | t.Fatalf("retried %v, want only this box's b.example.com, for v2 at epoch 5 of book bk", retried) |
| 610 | } |
| 611 | } |
| 612 | |
| 613 | // retryMemory is a restart memory that verifies retries against a fresh |
| 614 | // root key, returned for the test's console to sign with. |
| 615 | func retryMemory(t *testing.T) (*restartMemory, ed25519.PrivateKey) { |
| 616 | t.Helper() |
| 617 | pub, priv, err := ed25519.GenerateKey(nil) |
| 618 | if err != nil { |
| 619 | t.Fatal(err) |
| 620 | } |
| 621 | mem := newRestartMemory() |
| 622 | mem.pub = pub |
| 623 | return mem, priv |
| 624 | } |
| 625 | |
| 626 | // askRetry is the console's signed `carlos instances retry <host>` landing |
| 627 | // in the bucket (internal/console requestHostRetry). |
| 628 | func askRetry(t *testing.T, ctx context.Context, s release.Store, priv ed25519.PrivateKey, acct, app, host, build, book string, epoch uint64) string { |
| 629 | t.Helper() |
| 630 | wire, err := release.SignHostRetry(release.RootSigner(priv), acct, app, host, build, book, epoch, time.Now()) |
| 631 | if err != nil { |
| 632 | t.Fatal(err) |
| 633 | } |
| 634 | askSignedRetry(t, ctx, s, acct, app, host, wire) |
| 635 | return wire |
| 636 | } |
| 637 | |
| 638 | // askSignedRetry is a console's append of a signed grant to the host-retry |
| 639 | // log (release.AppendHostRetry). |
| 640 | func askSignedRetry(t *testing.T, ctx context.Context, s release.Store, acct, app, host, wire string) { |
| 641 | t.Helper() |
| 642 | if _, err := release.AppendHostRetry(ctx, s, acct, app, host, wire); err != nil { |
| 643 | t.Fatal(err) |
| 644 | } |
| 645 | } |
| 646 | |
| 647 | // rawRetry appends a retry entry to the log at key with no validation: an |
| 648 | // unsigned one in the host-retry log (any box can write it), or a retry in |
| 649 | // the restart-scope log as a console built from main before the host-retry |
| 650 | // log wrote it (legacyRetry), signed or, from an older one, not. |
| 651 | func rawRetry(t *testing.T, ctx context.Context, s release.Store, key, host, wire string) string { |
| 652 | t.Helper() |
| 653 | g := &release.ScopedRestartGeneration{Schema: 1} |
| 654 | raw, etag, err := s.Get(ctx, key) |
| 655 | switch { |
| 656 | case errors.Is(err, os.ErrNotExist): |
| 657 | etag = "" |
| 658 | case err != nil: |
| 659 | t.Fatal(err) |
| 660 | default: |
| 661 | if err := json.Unmarshal(raw, g); err != nil { |
| 662 | t.Fatal(err) |
| 663 | } |
| 664 | } |
| 665 | id := fmt.Sprintf("raw-%d-%d", len(g.Requests), time.Now().UnixNano()) |
| 666 | g.Requests = append(g.Requests, release.RestartRequest{ID: id, RequestedAt: time.Now().UTC().Format(time.RFC3339), |
| 667 | Scope: release.RestartScope{Kind: release.RestartScopeRetry, Value: host, Signed: wire}}) |
| 668 | body, err := json.Marshal(g) |
| 669 | if err != nil { |
| 670 | t.Fatal(err) |
| 671 | } |
| 672 | if _, err := s.Put(ctx, key, body, etag); err != nil { |
| 673 | t.Fatal(err) |
| 674 | } |
| 675 | return id |
| 676 | } |
| 677 | |
| 678 | // legacyRetry is a retry in the restart-scope log, as a console built from |
| 679 | // main before the host-retry log appended it. |
| 680 | func legacyRetry(t *testing.T, ctx context.Context, s release.Store, acct, app, host, wire string) { |
| 681 | t.Helper() |
| 682 | rawRetry(t, ctx, s, release.RestartScopeKey(acct, app), host, wire) |
| 683 | } |
| 684 | |
| 685 | // legacySignedRetry signs a grant as legacyRetry's console did. |
| 686 | func legacySignedRetry(t *testing.T, ctx context.Context, s release.Store, priv ed25519.PrivateKey, acct, app, host, build, book string, epoch uint64) { |
| 687 | t.Helper() |
| 688 | wire, err := release.SignHostRetry(release.RootSigner(priv), acct, app, host, build, book, epoch, time.Now()) |
| 689 | if err != nil { |
| 690 | t.Fatal(err) |
| 691 | } |
| 692 | legacyRetry(t, ctx, s, acct, app, host, wire) |
| 693 | } |
| 694 | |
| 695 | // Spec §11, Astra impl-6 finding 5: every box can write the restart-scope |
| 696 | // log, so only a grant the root signed for this entry's account, app and |
| 697 | // host reaches the manager. An unsigned entry (another box's, or an older |
| 698 | // console's), one signed by another key, one whose grant names another host |
| 699 | // or app than its entry, and any retry on a box with no root key are |
| 700 | // dropped with a line saying why. |
| 701 | func TestRestartLegDropsARetryThatIsNotTheRootsGrant(t *testing.T) { |
| 702 | ctx, s := genKit(t) |
| 703 | dd := t.TempDir() |
| 704 | routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")} |
| 705 | mem, priv := retryMemory(t) |
| 706 | _, other, _ := ed25519.GenerateKey(nil) |
| 707 | mem.retryHost = func(h, b, _ string, e uint64) bool { |
| 708 | t.Errorf("retryHost(%s, %s, %d) was called", h, b, e) |
| 709 | return true |
| 710 | } |
| 711 | bounce := func(h string) bool { t.Errorf("%s was bounced", h); return true } |
| 712 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline |
| 713 | |
| 714 | rawRetry(t, ctx, s, release.HostRetryKey("bab", "eleven"), "a.example.com", "") |
| 715 | askRetry(t, ctx, s, other, "bab", "eleven", "a.example.com", "v2", "bk", 5) |
| 716 | forB, _ := release.SignHostRetry(release.RootSigner(priv), "bab", "eleven", "b.example.com", "v2", "bk", 5, time.Now()) |
| 717 | askSignedRetry(t, ctx, s, "bab", "eleven", "a.example.com", forB) |
| 718 | forShop, _ := release.SignHostRetry(release.RootSigner(priv), "bab", "shop", "a.example.com", "v2", "bk", 5, time.Now()) |
| 719 | askSignedRetry(t, ctx, s, "bab", "eleven", "a.example.com", forShop) |
| 720 | |
| 721 | var logbuf bytes.Buffer |
| 722 | log.SetOutput(&logbuf) |
| 723 | defer log.SetOutput(os.Stderr) |
| 724 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 725 | if n := strings.Count(logbuf.String(), "agent: retry requested for a.example.com, dropped:"); n != 4 { |
| 726 | t.Fatalf("log = %q, want four dropped retries", logbuf.String()) |
| 727 | } |
| 728 | if !strings.Contains(logbuf.String(), "unsigned") { |
| 729 | t.Fatalf("log = %q, want the unsigned entry named as such", logbuf.String()) |
| 730 | } |
| 731 | |
| 732 | // A box with no root key cannot tell a grant from a forgery. |
| 733 | logbuf.Reset() |
| 734 | mem.pub = nil |
| 735 | askRetry(t, ctx, s, priv, "bab", "eleven", "a.example.com", "v2", "bk", 5) |
| 736 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 737 | if !strings.Contains(logbuf.String(), "agent: retry requested for a.example.com, dropped: this box has no root public key") { |
| 738 | t.Fatalf("log = %q, want the keyless drop named", logbuf.String()) |
| 739 | } |
| 740 | } |
| 741 | |
| 742 | // The grants against a real failure book (plan review findings 5 and 13): a |
| 743 | // grant whose epoch predates a second failure of the same build on the same |
| 744 | // host, a grant for another build, a grant for another book (final review |
| 745 | // round 1, finding 3), and a used grant replayed after the book is reopened |
| 746 | // each clear nothing; the grant for the current restriction clears it. |
| 747 | func TestRestartLegRetryGrantsAgainstTheBook(t *testing.T) { |
| 748 | ctx, s := genKit(t) |
| 749 | dd, bookDir := t.TempDir(), t.TempDir() |
| 750 | const host = "a.example.com" |
| 751 | routes := []registry.Route{execRoute(host, "bab", "eleven")} |
| 752 | book, err := rollout.OpenFailureBook(bookDir) |
| 753 | if err != nil { |
| 754 | t.Fatal(err) |
| 755 | } |
| 756 | mem, priv := retryMemory(t) |
| 757 | var cleared int |
| 758 | mem.retryHost = func(h, b, bk string, e uint64) bool { |
| 759 | rec, mark, _, _ := book.RetryHostAt("bab", "eleven", b, h, bk, e) |
| 760 | if rec || mark { |
| 761 | cleared++ |
| 762 | } |
| 763 | return rec || mark |
| 764 | } |
| 765 | bounce := func(h string) bool { t.Errorf("%s was bounced", h); return true } |
| 766 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline |
| 767 | held := func() bool { return book.Blocks("bab", "eleven", "v2", host, false) } |
| 768 | |
| 769 | book.Record(rollout.FailureRecord{Account: "bab", App: "eleven", Build: "v2", Host: host}) |
| 770 | first := book.RestrictionEpoch("bab", "eleven", "v2", host) |
| 771 | stale := askRetry(t, ctx, s, priv, "bab", "eleven", host, "v2", book.ID(), first) |
| 772 | // The same build fails again on the same host before the box reads it. |
| 773 | book.Record(rollout.FailureRecord{Account: "bab", App: "eleven", Build: "v2", Host: host, HeldDown: []string{host}}) |
| 774 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 775 | if cleared != 0 || !held() { |
| 776 | t.Fatal("a grant predating the second failure cleared it") |
| 777 | } |
| 778 | |
| 779 | now := book.RestrictionEpoch("bab", "eleven", "v2", host) |
| 780 | askRetry(t, ctx, s, priv, "bab", "eleven", host, "v9", book.ID(), now) |
| 781 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 782 | if cleared != 0 || !held() { |
| 783 | t.Fatal("a grant for another build cleared v2's hold") |
| 784 | } |
| 785 | |
| 786 | // Final review round 1, finding 3: the current epoch signed for another |
| 787 | // book (another box's, or this host's before a relocation) clears |
| 788 | // nothing, and nor does one that names no book (an older console). |
| 789 | askRetry(t, ctx, s, priv, "bab", "eleven", host, "v2", "another-book", now) |
| 790 | askRetry(t, ctx, s, priv, "bab", "eleven", host, "v2", "", now) |
| 791 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 792 | if cleared != 0 || !held() { |
| 793 | t.Fatal("a grant at the current epoch for another book, or for none, cleared v2's hold") |
| 794 | } |
| 795 | |
| 796 | used := askRetry(t, ctx, s, priv, "bab", "eleven", host, "v2", book.ID(), now) |
| 797 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 798 | if cleared != 1 || held() { |
| 799 | t.Fatalf("the current grant: %d clears, held %v; want one clear", cleared, held()) |
| 800 | } |
| 801 | |
| 802 | // The agent restarts; the retried canary fails again; the used grant |
| 803 | // and the stale one are replayed under new entry ids. |
| 804 | if book, err = rollout.OpenFailureBook(bookDir); err != nil { |
| 805 | t.Fatal(err) |
| 806 | } |
| 807 | book.Record(rollout.FailureRecord{Account: "bab", App: "eleven", Build: "v2", Host: host, HeldDown: []string{host}, Seq: book.Snapshot()}) |
| 808 | if !held() { |
| 809 | t.Fatal("the failure after the retry did not hold the host") |
| 810 | } |
| 811 | askSignedRetry(t, ctx, s, "bab", "eleven", host, used) |
| 812 | askSignedRetry(t, ctx, s, "bab", "eleven", host, stale) |
| 813 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 814 | if cleared != 1 || !held() { |
| 815 | t.Fatal("a replayed grant cleared the failure that followed it") |
| 816 | } |
| 817 | } |
| 818 | |
| 819 | // With no manager armed (flag off, or a box that declined or refused it) |
| 820 | // nothing is held: a retry an older console left in the restart-scope log |
| 821 | // is consumed, logged, and cycles nothing. Such a box does not read the |
| 822 | // host-retry log at all (hostRetryPass): nothing it holds could be |
| 823 | // cleared. |
| 824 | func TestRestartLegDropsARetryWithNoManager(t *testing.T) { |
| 825 | ctx, s := genKit(t) |
| 826 | dd := t.TempDir() |
| 827 | routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} |
| 828 | mem := newRestartMemory() |
| 829 | bounce := func(h string) bool { t.Errorf("%s was bounced", h); return true } |
| 830 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 831 | legacyRetry(t, ctx, s, "bab", "eleven", "a.example.com", "") |
| 832 | |
| 833 | var logbuf bytes.Buffer |
| 834 | log.SetOutput(&logbuf) |
| 835 | defer log.SetOutput(os.Stderr) |
| 836 | if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked { |
| 837 | t.Fatal("a retry with no manager asked for a root pass") |
| 838 | } |
| 839 | if got := logbuf.String(); !strings.Contains(got, "retry requested for a.example.com") || !strings.Contains(got, "rollout manager not armed") { |
| 840 | t.Fatalf("log = %q, want the dropped retry named", got) |
| 841 | } |
| 842 | // Consumed: the next pass does not see it again. |
| 843 | logbuf.Reset() |
| 844 | if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked { |
| 845 | t.Fatal("the second pass asked for a root pass") |
| 846 | } |
| 847 | if got := logbuf.String(); strings.Contains(got, "retry requested") { |
| 848 | t.Fatalf("the retry was seen again on the next pass: %q", got) |
| 849 | } |
| 850 | } |
| 851 | |
| 852 | // Flag-off behaviour is unchanged: a box with no retry path that gains an |
| 853 | // app, or first sees one, whose restart-scope log holds a legacy retry reads |
| 854 | // it as history and says nothing about it. (A retry asked for since the last |
| 855 | // look still gets its dropped line: TestRestartLegDropsARetryWithNoManager.) |
| 856 | func TestRestartLegSaysNothingOfAHistoricRetryOnABoxWithNoManager(t *testing.T) { |
| 857 | ctx, s := genKit(t) |
| 858 | dd := t.TempDir() |
| 859 | one := []registry.Route{execRoute("a.example.com", "bab", "eleven")} |
| 860 | both := append(slices.Clone(one), execRoute("c.example.com", "bab", "carloku")) |
| 861 | mem := newRestartMemory() |
| 862 | bounce := func(h string) bool { t.Errorf("%s was bounced", h); return true } |
| 863 | restartRequestLeg(ctx, one, s, dd, bounce, nil, mem) // baseline: carloku is not here yet |
| 864 | legacyRetry(t, ctx, s, "bab", "carloku", "c.example.com", "") |
| 865 | |
| 866 | var logbuf bytes.Buffer |
| 867 | log.SetOutput(&logbuf) |
| 868 | defer log.SetOutput(os.Stderr) |
| 869 | if asked := restartRequestLeg(ctx, both, s, dd, bounce, nil, mem); asked { |
| 870 | t.Fatal("a historic retry asked for a root pass") |
| 871 | } |
| 872 | if got := logbuf.String(); strings.Contains(got, "retry") { |
| 873 | t.Fatalf("a box with no manager logged about a historic retry: %q", got) |
| 874 | } |
| 875 | } |
| 876 | |
| 877 | // A retry names a host but is not a restart: the narrowing that resolves hosts |
| 878 | // must never address one for it, on the agent's half or the root pass's. |
| 879 | func TestScopedHostsAndRootPassStepOverARetry(t *testing.T) { |
| 880 | routes := titogoRoutes() |
| 881 | var logbuf bytes.Buffer |
| 882 | log.SetOutput(&logbuf) |
| 883 | defer log.SetOutput(os.Stderr) |
| 884 | got := scopedHosts(routes, [2]string{"bab", "titogo"}, []release.RestartRequest{ |
| 885 | {ID: "1", Scope: release.RestartScope{Kind: release.RestartScopeRetry, Value: "bill.example.com"}}, |
| 886 | }, func(registry.Route) bool { return true }) |
| 887 | if len(got) != 0 { |
| 888 | t.Fatalf("a retry addressed %v for a restart", got) |
| 889 | } |
| 890 | if strings.Contains(logbuf.String(), "does not understand") { |
| 891 | t.Fatalf("a retry was logged as an unknown selector: %q", logbuf.String()) |
| 892 | } |
| 893 | |
| 894 | ctx, s := genKit(t) |
| 895 | marks := restartMarksPath(t.TempDir()) |
| 896 | restartPass(ctx, routes, s, marks, (&recordingRestart{}).fn) // baseline |
| 897 | legacyRetry(t, ctx, s, "bab", "titogo", "bill.example.com", "") |
| 898 | rec := &recordingRestart{} |
| 899 | restartPass(ctx, routes, s, marks, rec.fn) |
| 900 | if len(rec.hosts) != 0 { |
| 901 | t.Fatalf("the root pass cycled %v for a retry", rec.hosts) |
| 902 | } |
| 903 | } |
| 904 | |
| 905 | // A retry and a real narrowed restart in one poll window: the retry goes to |
| 906 | // the manager, only the restart cycles, and root is asked for its pass. |
| 907 | func TestRestartLegSeparatesARetryFromARestartInOneWindow(t *testing.T) { |
| 908 | ctx, s := genKit(t) |
| 909 | dd := t.TempDir() |
| 910 | routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")} |
| 911 | mem, priv := retryMemory(t) |
| 912 | var retried, bounced []string |
| 913 | mem.retryHost = func(h, _, _ string, _ uint64) bool { retried = append(retried, h); return true } |
| 914 | bounce := func(h string) bool { bounced = append(bounced, h); return true } |
| 915 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline |
| 916 | askRetry(t, ctx, s, priv, "bab", "eleven", "a.example.com", "v2", "bk", 5) |
| 917 | askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeHost, "b.example.com") |
| 918 | if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); !asked { |
| 919 | t.Fatal("the restart in the window did not ask for a root pass") |
| 920 | } |
| 921 | if !slices.Equal(retried, []string{"a.example.com"}) || !slices.Equal(bounced, []string{"b.example.com"}) { |
| 922 | t.Fatalf("retried %v, bounced %v; want the retry to a.example.com and the bounce of b.example.com only", retried, bounced) |
| 923 | } |
| 924 | } |
| 925 | |
| 926 | // A retry for a host the manager holds nothing for is logged as such, not |
| 927 | // as a hold cleared. |
| 928 | func TestRestartLegLogsARetryWithNothingHeld(t *testing.T) { |
| 929 | ctx, s := genKit(t) |
| 930 | dd := t.TempDir() |
| 931 | routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")} |
| 932 | mem, priv := retryMemory(t) |
| 933 | mem.retryHost = func(string, string, string, uint64) bool { return false } |
| 934 | bounce := func(h string) bool { t.Errorf("%s was bounced", h); return true } |
| 935 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 936 | askRetry(t, ctx, s, priv, "bab", "eleven", "a.example.com", "v2", "bk", 5) |
| 937 | var logbuf bytes.Buffer |
| 938 | log.SetOutput(&logbuf) |
| 939 | defer log.SetOutput(os.Stderr) |
| 940 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 941 | got := logbuf.String() |
| 942 | if !strings.Contains(got, "retry for a.example.com: nothing cleared") || strings.Contains(got, "hold is cleared") { |
| 943 | t.Fatalf("log = %q", got) |
| 944 | } |
| 945 | } |
| 946 | |
| 947 | // Plan 9 (Astra), a console built from main before the host-retry log: its |
| 948 | // signed, book-bound grant in the restart-scope log is verified and applied |
| 949 | // with the same checks as one in the host-retry log, and an unsigned or |
| 950 | // foreign one there is dropped with a line and clears nothing. An unchanged |
| 951 | // log is not handled twice. |
| 952 | func TestRestartLegAppliesALegacySignedRetryWithTheSameChecks(t *testing.T) { |
| 953 | ctx, s := genKit(t) |
| 954 | dd := t.TempDir() |
| 955 | routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")} |
| 956 | mem, priv := retryMemory(t) |
| 957 | _, other, _ := ed25519.GenerateKey(nil) |
| 958 | var retried []string |
| 959 | mem.retryHost = func(h, b, bk string, e uint64) bool { |
| 960 | retried = append(retried, fmt.Sprintf("%s %s %s %d", h, b, bk, e)) |
| 961 | return true |
| 962 | } |
| 963 | bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true } |
| 964 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) // baseline |
| 965 | |
| 966 | legacySignedRetry(t, ctx, s, priv, "bab", "eleven", "b.example.com", "v2", "bk", 5) |
| 967 | legacyRetry(t, ctx, s, "bab", "eleven", "a.example.com", "") |
| 968 | legacySignedRetry(t, ctx, s, other, "bab", "eleven", "a.example.com", "v2", "bk", 5) |
| 969 | var logbuf bytes.Buffer |
| 970 | log.SetOutput(&logbuf) |
| 971 | defer log.SetOutput(os.Stderr) |
| 972 | if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked { |
| 973 | t.Fatal("a retry-only move asked for a root restart pass") |
| 974 | } |
| 975 | if !slices.Equal(retried, []string{"b.example.com v2 bk 5"}) { |
| 976 | t.Fatalf("retried %v, want the legacy signed grant for b.example.com only", retried) |
| 977 | } |
| 978 | got := logbuf.String() |
| 979 | if strings.Count(got, "agent: retry requested for a.example.com, dropped:") != 2 || !strings.Contains(got, "unsigned") { |
| 980 | t.Fatalf("log = %q, want the unsigned and the foreign legacy entries dropped", got) |
| 981 | } |
| 982 | |
| 983 | logbuf.Reset() |
| 984 | restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem) |
| 985 | if len(retried) != 1 || strings.Contains(logbuf.String(), "retry") { |
| 986 | t.Fatalf("an unchanged log was handled again: retried %v, log %q", retried, logbuf.String()) |
| 987 | } |
| 988 | } |
| 989 | |
| 990 | // Plan 9 (Astra), the ruling that grants are not seed-skipped: a grant |
| 991 | // written before an agent restart, and not yet applied, is applied after |
| 992 | // it, whichever log it rides, because the restart memory is rebuilt and a |
| 993 | // spent or stale grant is a no-op under the generation rule. A restart |
| 994 | // request written before the restart keeps today's seed-skip: it cycles |
| 995 | // nothing. The grants are handled once. |
| 996 | func TestARetryGrantWrittenBeforeAnAgentRestartIsApplied(t *testing.T) { |
| 997 | ctx, s := genKit(t) |
| 998 | reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db")) |
| 999 | if err != nil { |
| 1000 | t.Fatal(err) |
| 1001 | } |
| 1002 | defer reg.Close() |
| 1003 | routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")} |
| 1004 | for _, rt := range routes { |
| 1005 | if err := reg.Upsert(rt); err != nil { |
| 1006 | t.Fatal(err) |
| 1007 | } |
| 1008 | } |
| 1009 | dd := t.TempDir() |
| 1010 | first, priv := retryMemory(t) |
| 1011 | first.retryHost = func(h, b, _ string, e uint64) bool { |
| 1012 | t.Errorf("the first agent applied %s %s %d", h, b, e) |
| 1013 | return true |
| 1014 | } |
| 1015 | restartSeed(ctx, reg, s, dd, first) |
| 1016 | |
| 1017 | // The grants and a restart land; the agent restarts before its next |
| 1018 | // serial move. |
| 1019 | askRetry(t, ctx, s, priv, "bab", "eleven", "a.example.com", "v2", "bk", 5) |
| 1020 | legacySignedRetry(t, ctx, s, priv, "bab", "eleven", "b.example.com", "v3", "bk", 6) |
| 1021 | askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeHost, "a.example.com") |
| 1022 | |
| 1023 | second := newRestartMemory() |
| 1024 | second.pub = first.pub |
| 1025 | var retried []string |
| 1026 | second.retryHost = func(h, b, bk string, e uint64) bool { |
| 1027 | retried = append(retried, fmt.Sprintf("%s %s %s %d", h, b, bk, e)) |
| 1028 | return true |
| 1029 | } |
| 1030 | restartSeed(ctx, reg, s, dd, second) |
| 1031 | slices.Sort(retried) |
| 1032 | if !slices.Equal(retried, []string{"a.example.com v2 bk 5", "b.example.com v3 bk 6"}) { |
| 1033 | t.Fatalf("after the restart retried %v, want both grants written before it", retried) |
| 1034 | } |
| 1035 | bounce := func(h string) bool { t.Errorf("%s was bounced: restarted before the agent", h); return true } |
| 1036 | if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, second); asked { |
| 1037 | t.Fatal("the first move after the restart asked for a root pass") |
| 1038 | } |
| 1039 | if len(retried) != 2 { |
| 1040 | t.Fatalf("the grants were handled again: %v", retried) |
| 1041 | } |
| 1042 | } |
| 1043 | |
amadan