| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "path/filepath" |
| 8 | "reflect" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "sync/atomic" |
| 12 | "testing" |
| 13 | "time" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/appconfig" |
| 16 | "github.com/carlosframework/platform/internal/hostwrite" |
| 17 | "github.com/carlosframework/platform/internal/layout" |
| 18 | "github.com/carlosframework/platform/internal/provision" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | ) |
| 22 | |
| 23 | // resetLayoutGlobals puts the package's published layout state back the |
| 24 | // way a fresh process has it, before and after a test that touches it. |
| 25 | func resetLayoutGlobals(t *testing.T) { |
| 26 | t.Helper() |
| 27 | clear := func() { |
| 28 | armedLayoutPass.Store(nil) |
| 29 | layoutReload.Store(nil) |
| 30 | layoutIntentSeen.Store(nil) |
| 31 | layoutDefaultGate.Store(nil) |
| 32 | layoutDefaultSeen.Store(nil) |
| 33 | } |
| 34 | clear() |
| 35 | t.Cleanup(clear) |
| 36 | } |
| 37 | |
| 38 | func appendIntent(t *testing.T, s release.Store, mode, box string) { |
| 39 | t.Helper() |
| 40 | if _, err := release.AppendLayoutIntent(context.Background(), s, release.LayoutIntent{Mode: mode, Scope: release.LayoutScope{Box: box}}); err != nil { |
| 41 | t.Fatalf("AppendLayoutIntent: %v", err) |
| 42 | } |
| 43 | } |
| 44 | |
| 45 | func TestLayoutLegConvergesOnlyWhenTheIntentAppliesToThisBox(t *testing.T) { |
| 46 | resetLayoutGlobals(t) |
| 47 | s := release.NewDirStore(t.TempDir()) |
| 48 | ctx := context.Background() |
| 49 | var calls []layout.Mode |
| 50 | converge := func(_ context.Context, mode layout.Mode, _ bool) []layout.Result { |
| 51 | calls = append(calls, mode) |
| 52 | return nil |
| 53 | } |
| 54 | |
| 55 | layoutLegWith(ctx, s, "kass-1", converge) |
| 56 | if len(calls) != 0 { |
| 57 | t.Fatalf("converged with no intent: %v", calls) |
| 58 | } |
| 59 | if got := lastLayoutIntent(); got != "" { |
| 60 | t.Fatalf("intent seen with none recorded = %q", got) |
| 61 | } |
| 62 | |
| 63 | appendIntent(t, s, release.LayoutModeNested, "kass-2") |
| 64 | layoutLegWith(ctx, s, "kass-1", converge) |
| 65 | if len(calls) != 0 { |
| 66 | t.Fatalf("converged for another box's intent: %v", calls) |
| 67 | } |
| 68 | |
| 69 | appendIntent(t, s, release.LayoutModeFlat, "") |
| 70 | layoutLegWith(ctx, s, "kass-1", converge) |
| 71 | if !reflect.DeepEqual(calls, []layout.Mode{layout.ModeFlat}) { |
| 72 | t.Fatalf("calls = %v, want [flat]", calls) |
| 73 | } |
| 74 | if got := lastLayoutIntent(); got != release.LayoutModeFlat { |
| 75 | t.Fatalf("intent seen = %q, want flat", got) |
| 76 | } |
| 77 | |
| 78 | appendIntent(t, s, release.LayoutModeNested, "kass-1") |
| 79 | layoutLegWith(ctx, s, "kass-1", converge) |
| 80 | if !reflect.DeepEqual(calls, []layout.Mode{layout.ModeFlat, layout.ModeNested}) { |
| 81 | t.Fatalf("calls = %v, want [flat nested]", calls) |
| 82 | } |
| 83 | if got := lastLayoutIntent(); got != release.LayoutModeNested { |
| 84 | t.Fatalf("intent seen = %q, want nested", got) |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | // An intent this binary cannot read, or a request whose mode it does not |
| 89 | // know, moves nothing, and publishes layout.IntentUnreadable — overwriting |
| 90 | // whatever mode was last seen, since a read that failed just now is not |
| 91 | // proof the fleet still wants that mode. A later successful read replaces |
| 92 | // the marker the same way (TestStartupLayoutIntentPublishesUnreadable). |
| 93 | func TestLayoutLegDoesNothingWhenTheIntentIsUnreadable(t *testing.T) { |
| 94 | for _, tc := range []struct { |
| 95 | name string |
| 96 | body string |
| 97 | }{ |
| 98 | {"newer schema", `{"schema":2,"requests":[{"id":"a","mode":"nested"}]}`}, |
| 99 | {"unknown mode", `{"schema":1,"requests":[{"id":"a","mode":"sideways"}]}`}, |
| 100 | {"not json", `{`}, |
| 101 | } { |
| 102 | t.Run(tc.name, func(t *testing.T) { |
| 103 | resetLayoutGlobals(t) |
| 104 | noteLayoutIntent(release.LayoutModeFlat) |
| 105 | s := release.NewDirStore(t.TempDir()) |
| 106 | ctx := context.Background() |
| 107 | if _, err := s.Put(ctx, release.LayoutIntentKey(), []byte(tc.body), ""); err != nil { |
| 108 | t.Fatal(err) |
| 109 | } |
| 110 | called := false |
| 111 | layoutLegWith(ctx, s, "kass-1", func(context.Context, layout.Mode, bool) []layout.Result { |
| 112 | called = true |
| 113 | return nil |
| 114 | }) |
| 115 | if called { |
| 116 | t.Fatal("converged under an intent it could not read") |
| 117 | } |
| 118 | if got := lastLayoutIntent(); got != string(layout.IntentUnreadable) { |
| 119 | t.Fatalf("intent seen = %q, want unreadable (the last read failed, not the last mode read)", got) |
| 120 | } |
| 121 | }) |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | // With no pass armed (no activator on this box) the leg reads nothing. |
| 126 | func TestLayoutLegWithNoPassArmedReadsNothing(t *testing.T) { |
| 127 | resetLayoutGlobals(t) |
| 128 | runLayoutLeg(context.Background(), refusingStore{t: t}, "kass-1") |
| 129 | } |
| 130 | |
| 131 | // refusingStore fails the test on a read; every other method is the nil |
| 132 | // embedded Store's, which panics, which fails it too. |
| 133 | type refusingStore struct { |
| 134 | release.Store |
| 135 | t *testing.T |
| 136 | } |
| 137 | |
| 138 | func (r refusingStore) Get(context.Context, string) ([]byte, string, error) { |
| 139 | r.t.Fatal("the store was read") |
| 140 | return nil, "", nil |
| 141 | } |
| 142 | |
| 143 | // The leg never runs two passes at once, never makes the serial tick wait |
| 144 | // for one, and a move that lands mid-pass costs exactly one more pass |
| 145 | // however many moves there were. |
| 146 | func TestLayoutLegRunsOneAtATimeAndOnceMoreForAMoveMidPass(t *testing.T) { |
| 147 | started := make(chan struct{}, 8) |
| 148 | release1 := make(chan struct{}) |
| 149 | var runs, inFlight, most atomic.Int32 |
| 150 | leg := &layoutLeg{run: func(context.Context) { |
| 151 | n := inFlight.Add(1) |
| 152 | for { |
| 153 | m := most.Load() |
| 154 | if n <= m || most.CompareAndSwap(m, n) { |
| 155 | break |
| 156 | } |
| 157 | } |
| 158 | if runs.Add(1) == 1 { |
| 159 | started <- struct{}{} |
| 160 | <-release1 |
| 161 | } |
| 162 | inFlight.Add(-1) |
| 163 | }} |
| 164 | ctx := context.Background() |
| 165 | |
| 166 | returned := make(chan struct{}) |
| 167 | go func() { |
| 168 | leg.kick(ctx) |
| 169 | close(returned) |
| 170 | }() |
| 171 | select { |
| 172 | case <-returned: |
| 173 | case <-time.After(5 * time.Second): |
| 174 | t.Fatal("kick waited for the pass") |
| 175 | } |
| 176 | select { |
| 177 | case <-started: |
| 178 | case <-time.After(5 * time.Second): |
| 179 | t.Fatal("the first pass never started") |
| 180 | } |
| 181 | for i := 0; i < 3; i++ { |
| 182 | leg.kick(ctx) |
| 183 | } |
| 184 | close(release1) |
| 185 | leg.wait() |
| 186 | if got := runs.Load(); got != 2 { |
| 187 | t.Fatalf("passes = %d, want 2 (the first, and one for the moves mid-pass)", got) |
| 188 | } |
| 189 | if got := most.Load(); got != 1 { |
| 190 | t.Fatalf("%d passes ran at once", got) |
| 191 | } |
| 192 | |
| 193 | // Idle again: the next move runs one pass. |
| 194 | leg.kick(ctx) |
| 195 | leg.wait() |
| 196 | if got := runs.Load(); got != 3 { |
| 197 | t.Fatalf("passes after an idle kick = %d, want 3", got) |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | // A kick that lands between the runner's last look at the dirty flag and |
| 202 | // its going idle is taken back up by that runner, not lost. |
| 203 | func TestLayoutLegTakesUpAKickThatLandsAsItGoesIdle(t *testing.T) { |
| 204 | leg := &layoutLeg{run: func(context.Context) { t.Fatal("kick started a second runner") }} |
| 205 | leg.running.Store(true) // a runner past its last look at dirty |
| 206 | leg.kick(context.Background()) |
| 207 | if !leg.goIdle() { |
| 208 | t.Fatal("the runner went idle over a pending kick") |
| 209 | } |
| 210 | if !leg.running.Load() { |
| 211 | t.Fatal("the runner going again is not marked running") |
| 212 | } |
| 213 | leg.dirty.Store(false) |
| 214 | if leg.goIdle() { |
| 215 | t.Fatal("the runner went again with nothing pending") |
| 216 | } |
| 217 | if leg.running.Load() { |
| 218 | t.Fatal("an idle leg is marked running") |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | // Kicks from several goroutines at once, many times over: every one is |
| 223 | // answered and the leg ends idle with nothing pending (run under -race). |
| 224 | func TestLayoutLegNeverLosesAKick(t *testing.T) { |
| 225 | for i := 0; i < 200; i++ { |
| 226 | var runs atomic.Int32 |
| 227 | leg := &layoutLeg{run: func(context.Context) { runs.Add(1) }} |
| 228 | var wg sync.WaitGroup |
| 229 | for k := 0; k < 4; k++ { |
| 230 | wg.Add(1) |
| 231 | go func() { |
| 232 | defer wg.Done() |
| 233 | leg.kick(context.Background()) |
| 234 | }() |
| 235 | } |
| 236 | wg.Wait() |
| 237 | leg.wait() |
| 238 | if runs.Load() == 0 { |
| 239 | t.Fatal("four kicks ran no pass") |
| 240 | } |
| 241 | if leg.dirty.Load() { |
| 242 | t.Fatalf("iteration %d: a kick was left pending with the leg idle", i) |
| 243 | } |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | // Recovery runs after the startup unit sweep and before the activator |
| 248 | // starts, and it runs even when the sweep failed, told so. |
| 249 | func TestStartActivatorRecoversAfterTheSweepAndBeforeStart(t *testing.T) { |
| 250 | for _, tc := range []struct { |
| 251 | name string |
| 252 | sweepErr error |
| 253 | swept bool |
| 254 | }{ |
| 255 | {"sweep ok", nil, true}, |
| 256 | {"sweep failed", errors.New("channel down"), false}, |
| 257 | } { |
| 258 | t.Run(tc.name, func(t *testing.T) { |
| 259 | var events []string |
| 260 | var gotSwept bool |
| 261 | startActivator(context.Background(), |
| 262 | func(context.Context) error { events = append(events, "sweep"); return tc.sweepErr }, |
| 263 | func(_ context.Context, swept bool) { events = append(events, "recover"); gotSwept = swept }, |
| 264 | func() { events = append(events, "start") }) |
| 265 | if want := []string{"sweep", "recover", "start"}; !reflect.DeepEqual(events, want) { |
| 266 | t.Fatalf("order = %v, want %v", events, want) |
| 267 | } |
| 268 | if gotSwept != tc.swept { |
| 269 | t.Fatalf("swept = %v, want %v", gotSwept, tc.swept) |
| 270 | } |
| 271 | }) |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | // The intent recovery runs under at start: "" when none applies, the |
| 276 | // recorded mode when one does, and IntentUnreadable whenever the box |
| 277 | // cannot be sure, which holds a completed revert fenced. |
| 278 | func TestStartupLayoutIntent(t *testing.T) { |
| 279 | ctx := context.Background() |
| 280 | resetLayoutGlobals(t) |
| 281 | // Recorded intents are what this test is about; opt out of the nested |
| 282 | // default so "no object" reads as none (TestStartupAndMintReadTheDefault |
| 283 | // covers the default). |
| 284 | t.Setenv(layoutDefaultEnv, "flat") |
| 285 | if got := startupLayoutIntent(ctx, nil, errors.New("no bucket"), "kass-1"); got != layout.IntentUnreadable { |
| 286 | t.Fatalf("no store: %q", got) |
| 287 | } |
| 288 | if got := lastLayoutIntent(); got != string(layout.IntentUnreadable) { |
| 289 | t.Fatalf("intent seen after no store = %q, want unreadable", got) |
| 290 | } |
| 291 | s := release.NewDirStore(t.TempDir()) |
| 292 | if got := startupLayoutIntent(ctx, s, nil, "kass-1"); got != "" { |
| 293 | t.Fatalf("no object: %q", got) |
| 294 | } |
| 295 | if got := lastLayoutIntent(); got != "" { |
| 296 | t.Fatalf("intent seen after no object = %q, want none (a successful read replaces the marker)", got) |
| 297 | } |
| 298 | appendIntent(t, s, release.LayoutModeNested, "kass-2") |
| 299 | if got := startupLayoutIntent(ctx, s, nil, "kass-1"); got != "" { |
| 300 | t.Fatalf("another box's intent: %q", got) |
| 301 | } |
| 302 | if got := lastLayoutIntent(); got != "" { |
| 303 | t.Fatalf("intent seen after another box's intent = %q, want none", got) |
| 304 | } |
| 305 | appendIntent(t, s, release.LayoutModeNested, "") |
| 306 | if got := startupLayoutIntent(ctx, s, nil, "kass-1"); got != layout.ModeNested { |
| 307 | t.Fatalf("fleet intent: %q", got) |
| 308 | } |
| 309 | if got := lastLayoutIntent(); got != release.LayoutModeNested { |
| 310 | t.Fatalf("intent seen = %q", got) |
| 311 | } |
| 312 | for _, body := range []string{ |
| 313 | `{"schema":2,"requests":[]}`, |
| 314 | `{"schema":1,"requests":[{"id":"a","mode":"sideways"}]}`, |
| 315 | } { |
| 316 | bad := release.NewDirStore(t.TempDir()) |
| 317 | if _, err := bad.Put(ctx, release.LayoutIntentKey(), []byte(body), ""); err != nil { |
| 318 | t.Fatal(err) |
| 319 | } |
| 320 | if got := startupLayoutIntent(ctx, bad, nil, "kass-1"); got != layout.IntentUnreadable { |
| 321 | t.Fatalf("%s: %q", body, got) |
| 322 | } |
| 323 | // The prior call published "nested"; this failed read must overwrite |
| 324 | // it with the marker, not leave the stale mode standing. |
| 325 | if got := lastLayoutIntent(); got != string(layout.IntentUnreadable) { |
| 326 | t.Fatalf("%s: intent seen = %q, want unreadable to overwrite the last known mode", body, got) |
| 327 | } |
| 328 | noteLayoutIntent(release.LayoutModeNested) // restore, for the next iteration's overwrite check |
| 329 | } |
| 330 | } |
| 331 | |
| 332 | // Startup reads the intent only when a journal is there to recover. With |
| 333 | // none, recovery has nothing to do, so it runs under "" without touching |
| 334 | // the deploy store, and the published intent is left for the first serial |
| 335 | // leg to set. |
| 336 | func TestStartupRecoveryIntentReadsOnlyWithAJournal(t *testing.T) { |
| 337 | ctx := context.Background() |
| 338 | s := release.NewDirStore(t.TempDir()) |
| 339 | appendIntent(t, s, release.LayoutModeNested, "") |
| 340 | dd := t.TempDir() |
| 341 | opened := 0 |
| 342 | open := func(context.Context) (release.Store, error) { opened++; return s, nil } |
| 343 | |
| 344 | resetLayoutGlobals(t) |
| 345 | if got := startupRecoveryIntent(ctx, dd, open, "kass-1"); got != "" || opened != 0 { |
| 346 | t.Fatalf("no layout directory: intent %q, store opened %d times", got, opened) |
| 347 | } |
| 348 | dir := filepath.Join(dd, "layout") |
| 349 | if err := os.MkdirAll(dir, 0o700); err != nil { |
| 350 | t.Fatal(err) |
| 351 | } |
| 352 | if err := os.WriteFile(filepath.Join(dir, ".h.example.com.tmp-1"), nil, 0o600); err != nil { |
| 353 | t.Fatal(err) |
| 354 | } |
| 355 | if got := startupRecoveryIntent(ctx, dd, open, "kass-1"); got != "" || opened != 0 { |
| 356 | t.Fatalf("a temp file only: intent %q, store opened %d times", got, opened) |
| 357 | } |
| 358 | if m := layoutIntentSeen.Load(); m != nil { |
| 359 | t.Fatalf("published intent %q with no journal; the first serial leg sets it", *m) |
| 360 | } |
| 361 | |
| 362 | if err := os.WriteFile(provision.LayoutJournalPath(dd, "h.example.com"), []byte("{}"), 0o600); err != nil { |
| 363 | t.Fatal(err) |
| 364 | } |
| 365 | if got := startupRecoveryIntent(ctx, dd, open, "kass-1"); got != layout.ModeNested || opened != 1 { |
| 366 | t.Fatalf("with a journal: intent %q, store opened %d times", got, opened) |
| 367 | } |
| 368 | if got := lastLayoutIntent(); got != release.LayoutModeNested { |
| 369 | t.Fatalf("intent seen = %q", got) |
| 370 | } |
| 371 | } |
| 372 | |
| 373 | // TestStartupLayoutIntentPublishesUnreadable pins the fix on its own: a box |
| 374 | // whose every read fails at startup must report layout.IntentUnreadable, not |
| 375 | // "" (none) — recovery already holds a completed revert fenced under exactly |
| 376 | // that intent, and the published status must not contradict it. |
| 377 | func TestStartupLayoutIntentPublishesUnreadable(t *testing.T) { |
| 378 | resetLayoutGlobals(t) |
| 379 | ctx := context.Background() |
| 380 | if got := lastLayoutIntent(); got != "" { |
| 381 | t.Fatalf("intent seen before any read = %q, want none", got) |
| 382 | } |
| 383 | if got := startupLayoutIntent(ctx, nil, errors.New("no deploy store configured"), "kass-1"); got != layout.IntentUnreadable { |
| 384 | t.Fatalf("startupLayoutIntent = %q, want unreadable", got) |
| 385 | } |
| 386 | if got := lastLayoutIntent(); got != string(layout.IntentUnreadable) { |
| 387 | t.Fatalf("intent seen = %q, want unreadable, not none", got) |
| 388 | } |
| 389 | } |
| 390 | |
| 391 | // The layout leg rides the serial move after the schedule leg, and like |
| 392 | // every other leg it does not run on the boot seed or an unchanged serial. |
| 393 | func TestFastTickRunsTheLayoutLegAfterTheScheduleLeg(t *testing.T) { |
| 394 | env := newFastFixture(t) |
| 395 | var events []string |
| 396 | d := env.deps(func() { events = append(events, "pass") }) |
| 397 | d.scheduleLeg = func(context.Context) { events = append(events, "schedule") } |
| 398 | d.layoutLeg = func(context.Context) { events = append(events, "layout") } |
| 399 | now := time.Unix(1754640000, 0) |
| 400 | ctx := context.Background() |
| 401 | if err := release.BumpSerial(ctx, env.store, now); err != nil { |
| 402 | t.Fatal(err) |
| 403 | } |
| 404 | fastTick(ctx, d, now) |
| 405 | fastTick(ctx, d, now.Add(10*time.Second)) |
| 406 | if len(events) != 0 { |
| 407 | t.Fatalf("seed or unchanged serial ran %v", events) |
| 408 | } |
| 409 | if err := release.BumpSerial(ctx, env.store, now); err != nil { |
| 410 | t.Fatal(err) |
| 411 | } |
| 412 | fastTick(ctx, d, now.Add(20*time.Second)) |
| 413 | if want := []string{"pass", "schedule", "layout"}; !reflect.DeepEqual(events, want) { |
| 414 | t.Fatalf("legs = %v, want %v", events, want) |
| 415 | } |
| 416 | } |
| 417 | |
| 418 | // The pass the agent builds: moves serialize with config sync, the probe |
| 419 | // gate is the one handed in, routing reloads through the published edge |
| 420 | // (a no-op before the edge is up), awake hosts are deferred, and with no |
| 421 | // activator the fence is a nil interface rather than a nil pointer in one. |
| 422 | func TestNewLayoutPassWiring(t *testing.T) { |
| 423 | resetLayoutGlobals(t) |
| 424 | reg := restartTestRegistry(t) |
| 425 | dd := t.TempDir() |
| 426 | var gateCalls int |
| 427 | gate := func() error { gateCalls++; return nil } |
| 428 | p := newLayoutPass(reg, nil, dd, gate, true) |
| 429 | |
| 430 | if p.Fence != nil { |
| 431 | t.Fatalf("Fence = %#v with no activator, want a nil interface", p.Fence) |
| 432 | } |
| 433 | if !p.SweptAtStart || !p.DeferAwake || p.Registry != reg || p.DataDir != dd || p.Logf == nil { |
| 434 | t.Fatalf("pass = %+v", p) |
| 435 | } |
| 436 | if err := p.NestedOK(); err != nil || gateCalls != 1 { |
| 437 | t.Fatalf("NestedOK is not the gate handed in: err %v, calls %d", err, gateCalls) |
| 438 | } |
| 439 | |
| 440 | held := false |
| 441 | p.Serialize(func() { held = !configSyncMu.TryLock() }) |
| 442 | if !held { |
| 443 | t.Fatal("Serialize ran without the config-sync lock") |
| 444 | } |
| 445 | if !configSyncMu.TryLock() { |
| 446 | t.Fatal("Serialize left the config-sync lock held") |
| 447 | } |
| 448 | configSyncMu.Unlock() |
| 449 | |
| 450 | p.Reload() // nothing published yet: must not panic |
| 451 | reloads := 0 |
| 452 | f := func() { reloads++ } |
| 453 | layoutReload.Store(&f) |
| 454 | p.Reload() |
| 455 | if reloads != 1 { |
| 456 | t.Fatalf("reloads = %d, want 1", reloads) |
| 457 | } |
| 458 | |
| 459 | if q := newLayoutPass(reg, nil, dd, gate, false); q.SweptAtStart { |
| 460 | t.Fatal("SweptAtStart set after a failed sweep") |
| 461 | } |
| 462 | } |
| 463 | |
| 464 | // The one production SyncExec caller passes the journal predicate: a host |
| 465 | // with a layout journal gets no bundle, and its sibling does. |
| 466 | func TestSyncExecSerializedSkipsAHostWithALayoutJournal(t *testing.T) { |
| 467 | ctx := context.Background() |
| 468 | s := release.NewDirStore(t.TempDir()) |
| 469 | reg := restartTestRegistry(t) |
| 470 | dd := t.TempDir() |
| 471 | privText, _, err := appconfig.GenerateKey() |
| 472 | if err != nil { |
| 473 | t.Fatal(err) |
| 474 | } |
| 475 | priv, err := appconfig.ParsePrivateKey(privText, "test") |
| 476 | if err != nil { |
| 477 | t.Fatal(err) |
| 478 | } |
| 479 | if err := appconfig.SetEnv(ctx, s, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil { |
| 480 | t.Fatal(err) |
| 481 | } |
| 482 | held, sibling := "held.example.com", "sib.example.com" |
| 483 | for _, h := range []string{held, sibling} { |
| 484 | if err := reg.Upsert(registry.Route{Host: h, Kind: "instance", App: "eleven", Account: "bab", |
| 485 | Status: "ready", Socket: "/run/carlos/" + h + ".sock"}); err != nil { |
| 486 | t.Fatal(err) |
| 487 | } |
| 488 | } |
| 489 | journal := provision.LayoutJournalPath(dd, held) |
| 490 | if err := os.MkdirAll(filepath.Dir(journal), 0o700); err != nil { |
| 491 | t.Fatal(err) |
| 492 | } |
| 493 | if err := os.WriteFile(journal, []byte("{}"), 0o600); err != nil { |
| 494 | t.Fatal(err) |
| 495 | } |
| 496 | if err := syncExecSerialized(ctx, reg, s, priv, dd, nil); err != nil { |
| 497 | t.Fatal(err) |
| 498 | } |
| 499 | if _, err := os.Lstat(filepath.Join(dd, held+".config.env")); !os.IsNotExist(err) { |
| 500 | t.Fatalf("a held host got a bundle (err %v)", err) |
| 501 | } |
| 502 | if b, err := os.ReadFile(filepath.Join(dd, sibling+".config.env")); err != nil || string(b) != "LOG_LEVEL=debug\n" { |
| 503 | t.Fatalf("sibling bundle = %q, %v", b, err) |
| 504 | } |
| 505 | } |
| 506 | |
| 507 | // TestLayoutStatusReportWithNoPassArmedIsNil pins the snapshot writer's |
| 508 | // omission rule (R15): with no activator, and so no layout pass, there is |
| 509 | // nothing to move and nothing to report — the status snapshot must omit the |
| 510 | // block entirely rather than guess at a mode. |
| 511 | // With no pass armed (no activator on the box) the report is not nil but |
| 512 | // "unmanaged": rows judged by path shape, so the console's downgrade guard |
| 513 | // can still tell a nested database from a flat one, and a box that never |
| 514 | // arms an activator does not block its fleet's rollback by silence. |
| 515 | func TestLayoutStatusReportWithNoPassArmedIsUnmanagedByPathShape(t *testing.T) { |
| 516 | resetLayoutGlobals(t) |
| 517 | t.Setenv("CARLOS_DATA_DIR", "/data") |
| 518 | got := layoutStatusReport([]registry.Route{ |
| 519 | {Host: "nested.example.com", Kind: "instance", Hibernate: true, DBPath: provision.NestedDBPath("/data", "nested.example.com"), Socket: provision.NestedSocket("nested.example.com")}, |
| 520 | // Unflagged but under the instance tree: still nested by shape. |
| 521 | {Host: "unflagged.example.com", Kind: "instance", DBPath: provision.NestedDBPath("/data", "unflagged.example.com"), Socket: provision.NestedSocket("unflagged.example.com")}, |
| 522 | {Host: "flat.example.com", Kind: "instance", Hibernate: true, DBPath: provision.DefaultDBPath("/data", "flat.example.com"), Socket: provision.DefaultSocket("flat.example.com")}, |
| 523 | {Host: "external.example.com", Kind: "instance", Socket: provision.DefaultSocket("external.example.com")}, |
| 524 | {Host: "side.example.com", Kind: "sidecar"}, |
| 525 | {Host: "static.example.com", Kind: "static"}, |
| 526 | }) |
| 527 | if got == nil || got.Mode != LayoutModeUnmanaged { |
| 528 | t.Fatalf("report = %+v, want Mode unmanaged with no pass armed", got) |
| 529 | } |
| 530 | want := map[string]string{"nested.example.com": "nested", "unflagged.example.com": "nested", "flat.example.com": "flat", "external.example.com": "external"} |
| 531 | if len(got.Rows) != len(want) { |
| 532 | t.Fatalf("rows = %+v, want %v", got.Rows, want) |
| 533 | } |
| 534 | for host, layout := range want { |
| 535 | if got.Rows[host].Layout != layout { |
| 536 | t.Errorf("%s = %+v, want %s", host, got.Rows[host], layout) |
| 537 | } |
| 538 | } |
| 539 | } |
| 540 | |
| 541 | // A host mid-move (journal) or held after a refused move (hold marker) is |
| 542 | // fenced on an unmanaged box too, whatever its row's paths say: the |
| 543 | // files may be between two directories, and the console's downgrade guard |
| 544 | // must not read the flat-pathed row as flat. |
| 545 | func TestUnmanagedLayoutReportFencesAJournalledOrHeldHost(t *testing.T) { |
| 546 | resetLayoutGlobals(t) |
| 547 | dd := t.TempDir() |
| 548 | t.Setenv("CARLOS_DATA_DIR", dd) |
| 549 | if err := os.MkdirAll(filepath.Join(dd, "layout"), 0o700); err != nil { |
| 550 | t.Fatal(err) |
| 551 | } |
| 552 | if err := os.WriteFile(provision.LayoutJournalPath(dd, "moving.example.com"), []byte("{}"), 0o600); err != nil { |
| 553 | t.Fatal(err) |
| 554 | } |
| 555 | if err := os.WriteFile(filepath.Join(dd, "layout", "held.example.com.hold"), []byte("{}"), 0o600); err != nil { |
| 556 | t.Fatal(err) |
| 557 | } |
| 558 | flat := func(host string) registry.Route { |
| 559 | return registry.Route{Host: host, Kind: "instance", Hibernate: true, DBPath: provision.DefaultDBPath(dd, host), Socket: provision.DefaultSocket(host)} |
| 560 | } |
| 561 | got := layoutStatusReport([]registry.Route{flat("moving.example.com"), flat("held.example.com"), flat("flat.example.com")}) |
| 562 | want := map[string]string{"moving.example.com": "fenced", "held.example.com": "fenced", "flat.example.com": "flat"} |
| 563 | if len(got.Rows) != len(want) { |
| 564 | t.Fatalf("rows = %+v, want %v", got.Rows, want) |
| 565 | } |
| 566 | for host, layout := range want { |
| 567 | if got.Rows[host].Layout != layout { |
| 568 | t.Errorf("%s = %+v, want %s", host, got.Rows[host], layout) |
| 569 | } |
| 570 | } |
| 571 | } |
| 572 | |
| 573 | // mintLayoutIntent (ReconcileConfig.Layout for this box): an absent intent |
| 574 | // object is "" with no error, so mints proceed flat; only a real read |
| 575 | // failure, or a request this binary cannot read, is an error that defers |
| 576 | // them. |
| 577 | func TestMintLayoutIntentAbsentIsFlatAndUnreadableIsAnError(t *testing.T) { |
| 578 | ctx := context.Background() |
| 579 | s := release.NewDirStore(t.TempDir()) |
| 580 | if mode, err := mintLayoutIntent(ctx, s); err != nil || mode != "" { |
| 581 | t.Fatalf("absent object = %q %v, want \"\" and no error", mode, err) |
| 582 | } |
| 583 | if _, err := s.Put(ctx, release.LayoutIntentKey(), []byte(`{"schema":2,"requests":[]}`), ""); err != nil { |
| 584 | t.Fatal(err) |
| 585 | } |
| 586 | if _, err := mintLayoutIntent(ctx, s); err == nil { |
| 587 | t.Fatal("a newer schema must be an error, so the reconciler defers rather than mints flat") |
| 588 | } |
| 589 | } |
| 590 | |
| 591 | // TestLayoutStatusReportModeFollowsTheLastIntentSeen pins the three-way |
| 592 | // Mode mapping the amendment (R15) puts on the report, distinct from the |
| 593 | // brief's original two-word doc: "none" before any intent is recorded, the |
| 594 | // mode itself once one is, and "unreadable" when the last read of it |
| 595 | // failed — never silently downgraded to "none", which would read as "this |
| 596 | // box wants flat" when the box in fact could not tell. |
| 597 | func TestLayoutStatusReportModeFollowsTheLastIntentSeen(t *testing.T) { |
| 598 | resetLayoutGlobals(t) |
| 599 | dd, sd := t.TempDir(), t.TempDir() |
| 600 | armedLayoutPass.Store(&layout.Pass{DataDir: dd, SocketDir: sd}) |
| 601 | |
| 602 | if got := layoutStatusReport(nil); got == nil || got.Mode != "none" { |
| 603 | t.Fatalf("mode with no intent seen = %+v, want none", got) |
| 604 | } |
| 605 | noteLayoutIntent(release.LayoutModeFlat) |
| 606 | if got := layoutStatusReport(nil); got == nil || got.Mode != "flat" { |
| 607 | t.Fatalf("mode after a flat intent = %+v, want flat", got) |
| 608 | } |
| 609 | noteLayoutIntent(release.LayoutModeNested) |
| 610 | if got := layoutStatusReport(nil); got == nil || got.Mode != "nested" { |
| 611 | t.Fatalf("mode after a nested intent = %+v, want nested", got) |
| 612 | } |
| 613 | noteLayoutIntent(string(layout.IntentUnreadable)) |
| 614 | if got := layoutStatusReport(nil); got == nil || got.Mode != "unreadable" { |
| 615 | t.Fatalf("mode after an unreadable read = %+v, want unreadable", got) |
| 616 | } |
| 617 | } |
| 618 | |
| 619 | // TestLayoutStatusReportRowsCoverOnlyInstanceKindRoutes pins R15's row |
| 620 | // selection: an ActivatorManaged row goes through the pass's own Classify |
| 621 | // (proven here by a row shaped to classify as foreign, so the reason comes |
| 622 | // through too), an unflagged socket instance route reports "external", and |
| 623 | // everything else — a sidecar, a static route, a service route, and an |
| 624 | // instance route with no socket at all — carries no row, because the |
| 625 | // report is not the place any of those are decided. |
| 626 | func TestLayoutStatusReportRowsCoverOnlyInstanceKindRoutes(t *testing.T) { |
| 627 | resetLayoutGlobals(t) |
| 628 | dd, sd := t.TempDir(), t.TempDir() |
| 629 | armedLayoutPass.Store(&layout.Pass{DataDir: dd, SocketDir: sd}) |
| 630 | noteLayoutIntent(release.LayoutModeNested) |
| 631 | |
| 632 | routes := []registry.Route{ |
| 633 | // ActivatorManaged, but its paths match neither default — Classify |
| 634 | // must be the thing deciding this, not a guess made here. |
| 635 | {Host: "managed.example.com", Kind: "instance", Hibernate: true, |
| 636 | Socket: "/elsewhere/managed.sock", DBPath: "/elsewhere/managed.db", Status: "ready"}, |
| 637 | // Hibernate=false with a socket: externally supervised. |
| 638 | {Host: "external.example.com", Kind: "instance", Hibernate: false, Socket: "/elsewhere/external.sock"}, |
| 639 | // A sidecar, explicitly out of scope (R15, finding S6). |
| 640 | {Host: "side.example.com", Kind: "sidecar"}, |
| 641 | // Not instance-kind at all. |
| 642 | {Host: "static.example.com", Kind: "static"}, |
| 643 | {Host: "svc.example.com", Kind: "service"}, |
| 644 | // Instance-kind but no socket and not hibernate-flagged — neither |
| 645 | // managed nor the "unflagged socket" shape. |
| 646 | {Host: "nosock.example.com", Kind: "instance"}, |
| 647 | } |
| 648 | |
| 649 | report := layoutStatusReport(routes) |
| 650 | if report == nil { |
| 651 | t.Fatal("report = nil, want one with a pass armed") |
| 652 | } |
| 653 | if report.Mode != "nested" { |
| 654 | t.Errorf("mode = %q, want nested", report.Mode) |
| 655 | } |
| 656 | if len(report.Rows) != 2 { |
| 657 | t.Fatalf("rows = %+v, want exactly managed and external", report.Rows) |
| 658 | } |
| 659 | managed := report.Rows["managed.example.com"] |
| 660 | if managed.Layout != "foreign" || managed.Reason == "" { |
| 661 | t.Errorf("managed row = %+v, want foreign with a reason from Classify", managed) |
| 662 | } |
| 663 | external := report.Rows["external.example.com"] |
| 664 | if external.Layout != "external" || external.Reason != "" { |
| 665 | t.Errorf("external row = %+v, want external with no reason", external) |
| 666 | } |
| 667 | for _, host := range []string{"side.example.com", "static.example.com", "svc.example.com", "nosock.example.com"} { |
| 668 | if _, ok := report.Rows[host]; ok { |
| 669 | t.Errorf("%s has a row, want none", host) |
| 670 | } |
| 671 | } |
| 672 | } |
| 673 | |
| 674 | // Landing follow-up 3: a held host makes the leg come back on its own, so |
| 675 | // an operator who clears the reason on a quiet fleet waits awakeRetry and |
| 676 | // not the next serial move; an all-quiet result does not. |
| 677 | func TestLogLayoutResultsRetriesForHeldAndAwake(t *testing.T) { |
| 678 | if logLayoutResults([]layout.Result{{Host: "h", Outcome: "migrated"}, {Host: "i", Outcome: "unchanged"}}) { |
| 679 | t.Fatal("retry asked with nothing pending") |
| 680 | } |
| 681 | if !logLayoutResults([]layout.Result{{Host: "h", Outcome: "held", Reason: "x has 2 names"}}) { |
| 682 | t.Fatal("no retry for a held host") |
| 683 | } |
| 684 | if !logLayoutResults([]layout.Result{{Host: "h", Outcome: "deferred", Reason: "awake (stopped for the move in 10m0s unless it sleeps first)"}}) { |
| 685 | t.Fatal("no retry for an awake host") |
| 686 | } |
| 687 | } |
| 688 | |
| 689 | // A hold marker, like a journal, is something startup acts on, so it too |
| 690 | // makes the intent worth reading before the activator starts. |
| 691 | func TestStartupRecoveryIntentReadsWithAHoldMarker(t *testing.T) { |
| 692 | resetLayoutGlobals(t) |
| 693 | dd := t.TempDir() |
| 694 | opened := false |
| 695 | open := func(context.Context) (release.Store, error) { |
| 696 | opened = true |
| 697 | s := release.NewDirStore(t.TempDir()) |
| 698 | appendIntent(t, s, release.LayoutModeNested, "") |
| 699 | return s, nil |
| 700 | } |
| 701 | if err := os.MkdirAll(filepath.Join(dd, "layout"), 0o700); err != nil { |
| 702 | t.Fatal(err) |
| 703 | } |
| 704 | if err := os.WriteFile(filepath.Join(dd, "layout", "h.example.com.hold"), []byte(`{"host":"h.example.com","reason":"x"}`), 0o600); err != nil { |
| 705 | t.Fatal(err) |
| 706 | } |
| 707 | if got := startupRecoveryIntent(context.Background(), dd, open, "kass-1"); got != layout.ModeNested || !opened { |
| 708 | t.Fatalf("intent = %q opened=%v, want nested read for a hold marker", got, opened) |
| 709 | } |
| 710 | } |
| 711 | |
| 712 | func TestFastTickLayoutPrecedesClaimSerialNotify(t *testing.T) { |
| 713 | t.Parallel() |
| 714 | for _, occupied := range []bool{false, true} { |
| 715 | t.Run(map[bool]string{false: "empty", true: "full"}[occupied], func(t *testing.T) { |
| 716 | env := newFastFixture(t) |
| 717 | notify := make(chan struct{}, 1) |
| 718 | if occupied { |
| 719 | notify <- struct{}{} |
| 720 | } |
| 721 | wantBefore := len(notify) |
| 722 | d := env.deps(func() {}) |
| 723 | d.serialNotify = notify |
| 724 | layoutCalls := 0 |
| 725 | d.layoutLeg = func(context.Context) { |
| 726 | layoutCalls++ |
| 727 | if len(notify) != wantBefore { |
| 728 | t.Error("claim notification preceded layout leg") |
| 729 | } |
| 730 | } |
| 731 | now := time.Unix(1754640000, 0) |
| 732 | if err := release.BumpSerial(t.Context(), env.store, now); err != nil { |
| 733 | t.Fatal(err) |
| 734 | } |
| 735 | fastTick(t.Context(), d, now) |
| 736 | fastTick(t.Context(), d, now.Add(10*time.Second)) |
| 737 | if layoutCalls != 0 || len(notify) != wantBefore { |
| 738 | t.Fatal("seed or unchanged serial ran layout/claim notification") |
| 739 | } |
| 740 | if err := release.BumpSerial(t.Context(), env.store, now); err != nil { |
| 741 | t.Fatal(err) |
| 742 | } |
| 743 | done := make(chan struct{}) |
| 744 | go func() { |
| 745 | fastTick(t.Context(), d, now.Add(20*time.Second)) |
| 746 | close(done) |
| 747 | }() |
| 748 | select { |
| 749 | case <-done: |
| 750 | case <-time.After(5 * time.Second): |
| 751 | t.Fatal("serial notification blocked fast tick") |
| 752 | } |
| 753 | if layoutCalls != 1 || len(notify) != 1 { |
| 754 | t.Fatalf("layout calls=%d notifications=%d", layoutCalls, len(notify)) |
| 755 | } |
| 756 | }) |
| 757 | } |
| 758 | } |
| 759 | |
| 760 | func TestSyncExecSerializedFencedClaimHost(t *testing.T) { |
| 761 | t.Parallel() |
| 762 | reg := restartTestRegistry(t) |
| 763 | store := release.NewDirStore(t.TempDir()) |
| 764 | dd := t.TempDir() |
| 765 | host := "held.example.com" |
| 766 | if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Account: "bab", App: "app", Status: "ready", Socket: "/unused.sock"}); err != nil { |
| 767 | t.Fatal(err) |
| 768 | } |
| 769 | journal := provision.LayoutJournalPath(dd, host) |
| 770 | if err := os.MkdirAll(filepath.Dir(journal), 0700); err != nil { |
| 771 | t.Fatal(err) |
| 772 | } |
| 773 | if err := os.WriteFile(journal, []byte("{}"), 0600); err != nil { |
| 774 | t.Fatal(err) |
| 775 | } |
| 776 | writes := hostwrite.Config{Enabled: true} |
| 777 | if err := syncExecSerialized(t.Context(), reg, store, nil, dd, nil, writes); err != nil { |
| 778 | t.Fatalf("fenced host required a claim provider: %v", err) |
| 779 | } |
| 780 | if err := os.Remove(journal); err != nil { |
| 781 | t.Fatal(err) |
| 782 | } |
| 783 | if err := syncExecSerialized(t.Context(), reg, store, nil, dd, nil, writes); !errors.Is(err, hostwrite.ErrUnavailable) { |
| 784 | t.Fatalf("unfenced host lost its host-write option: %v", err) |
| 785 | } |
| 786 | } |
| 787 | |
| 788 | // armDefaultGate makes the default's readiness answer what the test says. |
| 789 | func armDefaultGate(t *testing.T, err error) { |
| 790 | t.Helper() |
| 791 | gate := func() error { return err } |
| 792 | layoutDefaultGate.Store(&gate) |
| 793 | } |
| 794 | |
| 795 | // defaultLayoutIntent (design §8.6, "Nested by default") is nested only |
| 796 | // when every readiness rule holds; each rule that fails names itself. |
| 797 | func TestDefaultLayoutIntentRules(t *testing.T) { |
| 798 | resetLayoutGlobals(t) |
| 799 | prev := layoutBuild |
| 800 | t.Cleanup(func() { layoutBuild = prev }) |
| 801 | layoutBuild = func() string { return "v0.25.0" } |
| 802 | |
| 803 | // No activator armed: not ready, says so. |
| 804 | t.Setenv(layoutDefaultEnv, "") |
| 805 | if mode, reason := defaultLayoutIntent(); mode != "" || !strings.Contains(reason, "no activator") { |
| 806 | t.Fatalf("no gate = %q %q", mode, reason) |
| 807 | } |
| 808 | // Gate refusing: not ready, with the gate's reason. |
| 809 | armDefaultGate(t, errors.New("instance control socket not listening")) |
| 810 | if mode, reason := defaultLayoutIntent(); mode != "" || !strings.Contains(reason, "isolation not ready: instance control socket not listening") { |
| 811 | t.Fatalf("gate refusing = %q %q", mode, reason) |
| 812 | } |
| 813 | // Ready: nested by default. |
| 814 | armDefaultGate(t, nil) |
| 815 | if mode, reason := defaultLayoutIntent(); mode != release.LayoutModeNested || reason != "" { |
| 816 | t.Fatalf("ready = %q %q, want nested", mode, reason) |
| 817 | } |
| 818 | // The deployment opt-out wins over readiness. |
| 819 | t.Setenv(layoutDefaultEnv, "flat") |
| 820 | if mode, reason := defaultLayoutIntent(); mode != "" || reason != "" { |
| 821 | t.Fatalf("CARLOS_LAYOUT_DEFAULT=flat = %q %q, want none with no reason", mode, reason) |
| 822 | } |
| 823 | t.Setenv(layoutDefaultEnv, " Nested ") |
| 824 | if mode, _ := defaultLayoutIntent(); mode != release.LayoutModeNested { |
| 825 | t.Fatalf("CARLOS_LAYOUT_DEFAULT=' Nested ' = %q, want nested (folded)", mode) |
| 826 | } |
| 827 | // A typo stays flat and says so. |
| 828 | t.Setenv(layoutDefaultEnv, "nestd") |
| 829 | if mode, reason := defaultLayoutIntent(); mode != "" || !strings.Contains(reason, "CARLOS_LAYOUT_DEFAULT=nestd is not nested or flat") { |
| 830 | t.Fatalf("typo = %q %q", mode, reason) |
| 831 | } |
| 832 | // A release below the floor is not ready, however the gate answers; |
| 833 | // an unplaceable build (this code, running) passes the floor. |
| 834 | t.Setenv(layoutDefaultEnv, "nested") |
| 835 | layoutBuild = func() string { return "v0.24.2" } |
| 836 | if mode, reason := defaultLayoutIntent(); mode != "" || !strings.Contains(reason, "runs v0.24.2, below v0.25.0") { |
| 837 | t.Fatalf("below floor = %q %q", mode, reason) |
| 838 | } |
| 839 | layoutBuild = func() string { return "abc1234" } |
| 840 | if mode, _ := defaultLayoutIntent(); mode != release.LayoutModeNested { |
| 841 | t.Fatalf("dev build = %q, want nested", mode) |
| 842 | } |
| 843 | } |
| 844 | |
| 845 | // The serial leg under no recorded intent: a ready box converges nested, |
| 846 | // never forced, and status says nested by default; a box that is not ready |
| 847 | // converges nothing and status says why; a recorded flat intent is the |
| 848 | // opt-out and wins; a recorded nested intent is not "by default". |
| 849 | func TestLayoutLegUsesTheDefaultWhenNoIntentIsRecorded(t *testing.T) { |
| 850 | resetLayoutGlobals(t) |
| 851 | armedLayoutPass.Store(&layout.Pass{DataDir: t.TempDir(), SocketDir: t.TempDir()}) |
| 852 | t.Setenv(layoutDefaultEnv, "") |
| 853 | prev := layoutBuild |
| 854 | t.Cleanup(func() { layoutBuild = prev }) |
| 855 | layoutBuild = func() string { return "v0.25.0" } |
| 856 | s := release.NewDirStore(t.TempDir()) |
| 857 | ctx := context.Background() |
| 858 | type call struct { |
| 859 | mode layout.Mode |
| 860 | force bool |
| 861 | } |
| 862 | var calls []call |
| 863 | converge := func(_ context.Context, mode layout.Mode, force bool) []layout.Result { |
| 864 | calls = append(calls, call{mode, force}) |
| 865 | return nil |
| 866 | } |
| 867 | |
| 868 | armDefaultGate(t, errors.New("probe: pid1 fail")) |
| 869 | layoutLegWith(ctx, s, "kass-1", converge) |
| 870 | if len(calls) != 0 { |
| 871 | t.Fatalf("converged while not ready: %v", calls) |
| 872 | } |
| 873 | if got := layoutStatusReport(nil); got.Mode != "none" || got.Default || !strings.Contains(got.DefaultBlocked, "isolation not ready: probe: pid1 fail") { |
| 874 | t.Fatalf("report while not ready = %+v", got) |
| 875 | } |
| 876 | |
| 877 | armDefaultGate(t, nil) |
| 878 | layoutLegWith(ctx, s, "kass-1", converge) |
| 879 | if !reflect.DeepEqual(calls, []call{{layout.ModeNested, false}}) { |
| 880 | t.Fatalf("calls = %v, want one unforced nested converge", calls) |
| 881 | } |
| 882 | if got := layoutStatusReport(nil); got.Mode != "nested" || !got.Default || got.DefaultBlocked != "" { |
| 883 | t.Fatalf("report when nested by default = %+v", got) |
| 884 | } |
| 885 | |
| 886 | // The opt-out: a recorded flat intent wins over a ready box. |
| 887 | appendIntent(t, s, release.LayoutModeFlat, "") |
| 888 | layoutLegWith(ctx, s, "kass-1", converge) |
| 889 | if len(calls) != 2 || calls[1].mode != layout.ModeFlat { |
| 890 | t.Fatalf("calls after a flat intent = %v, want a flat converge", calls) |
| 891 | } |
| 892 | if got := layoutStatusReport(nil); got.Mode != "flat" || got.Default || got.DefaultBlocked != "" { |
| 893 | t.Fatalf("report under a recorded flat intent = %+v", got) |
| 894 | } |
| 895 | |
| 896 | // A recorded nested intent is nested on purpose, not by default. |
| 897 | appendIntent(t, s, release.LayoutModeNested, "kass-1") |
| 898 | layoutLegWith(ctx, s, "kass-1", converge) |
| 899 | if got := layoutStatusReport(nil); got.Mode != "nested" || got.Default { |
| 900 | t.Fatalf("report under a recorded nested intent = %+v, want nested and not default", got) |
| 901 | } |
| 902 | } |
| 903 | |
| 904 | // Startup recovery and the reconciler's mint read the same default: with |
| 905 | // no intent and a ready box both answer nested; with no intent and a box |
| 906 | // that is not ready both answer none; CARLOS_LAYOUT_DEFAULT=flat answers |
| 907 | // none on a ready box. |
| 908 | func TestStartupAndMintReadTheDefault(t *testing.T) { |
| 909 | resetLayoutGlobals(t) |
| 910 | t.Setenv(layoutDefaultEnv, "") |
| 911 | prev := layoutBuild |
| 912 | t.Cleanup(func() { layoutBuild = prev }) |
| 913 | layoutBuild = func() string { return "v0.25.0" } |
| 914 | ctx := context.Background() |
| 915 | s := release.NewDirStore(t.TempDir()) |
| 916 | |
| 917 | // Startup never asks the gate: under the nested default it runs |
| 918 | // recovery fail-closed (IntentUnreadable keeps holds and journals), |
| 919 | // whatever the gate would say, and status says why. |
| 920 | armDefaultGate(t, errors.New("not yet run")) |
| 921 | if got := startupLayoutIntent(ctx, s, nil, "kass-1"); got != layout.IntentUnreadable { |
| 922 | t.Fatalf("startup, gate not ready = %q, want unreadable (holds kept)", got) |
| 923 | } |
| 924 | if d := lastLayoutDefault(); d.Applied || !strings.Contains(d.Blocked, "agent starting: holds and journals kept") { |
| 925 | t.Fatalf("default state at start = %+v", d) |
| 926 | } |
| 927 | armDefaultGate(t, nil) |
| 928 | if got := startupLayoutIntent(ctx, s, nil, "kass-1"); got != layout.IntentUnreadable { |
| 929 | t.Fatalf("startup, gate ready = %q, want unreadable: startup does not trust the gate", got) |
| 930 | } |
| 931 | // The mint asks the gate; not ready is flat... |
| 932 | armDefaultGate(t, errors.New("not yet run")) |
| 933 | if mode, err := mintLayoutIntent(ctx, s); err != nil || mode != "" { |
| 934 | t.Fatalf("mint, not ready = %q %v, want none", mode, err) |
| 935 | } |
| 936 | // ...unless the box already holds an activator-managed nested row, |
| 937 | // which is proof it has moved: a fresh row beside it is minted nested |
| 938 | // before the gate's first answer. A nested row that is not managed, |
| 939 | // or flat rows, are not that proof. |
| 940 | t.Setenv("CARLOS_DATA_DIR", "/data") |
| 941 | nestedManaged := registry.Route{Host: "a.example.com", Kind: "instance", Hibernate: true, DBPath: provision.NestedDBPath("/data", "a.example.com"), Socket: provision.NestedSocket("a.example.com")} |
| 942 | nestedUnflagged := registry.Route{Host: "u.example.com", Kind: "instance", DBPath: provision.NestedDBPath("/data", "u.example.com"), Socket: provision.NestedSocket("u.example.com")} |
| 943 | flat := registry.Route{Host: "f.example.com", Kind: "instance", Hibernate: true, DBPath: provision.DefaultDBPath("/data", "f.example.com"), Socket: provision.DefaultSocket("f.example.com")} |
| 944 | rows := func(rs ...registry.Route) func() ([]registry.Route, error) { |
| 945 | return func() ([]registry.Route, error) { return rs, nil } |
| 946 | } |
| 947 | if mode, err := mintLayoutIntentWith(ctx, s, rows(flat, nestedUnflagged)); err != nil || mode != "" { |
| 948 | t.Fatalf("mint, not ready, no managed nested row = %q %v, want none", mode, err) |
| 949 | } |
| 950 | if mode, err := mintLayoutIntentWith(ctx, s, rows(flat, nestedManaged)); err != nil || mode != release.LayoutModeNested { |
| 951 | t.Fatalf("mint, not ready, a managed nested row = %q %v, want nested", mode, err) |
| 952 | } |
| 953 | if _, err := mintLayoutIntentWith(ctx, s, func() ([]registry.Route, error) { return nil, errors.New("registry locked") }); err == nil { |
| 954 | t.Fatal("a registry that cannot be listed must defer the mint, not guess") |
| 955 | } |
| 956 | armDefaultGate(t, nil) |
| 957 | if mode, err := mintLayoutIntent(ctx, s); err != nil || mode != release.LayoutModeNested { |
| 958 | t.Fatalf("mint, ready = %q %v, want nested", mode, err) |
| 959 | } |
| 960 | // The opt-out: startup drops holds as a recorded flat would, and the |
| 961 | // mint is flat even beside a nested row. |
| 962 | t.Setenv(layoutDefaultEnv, "flat") |
| 963 | if got := startupLayoutIntent(ctx, s, nil, "kass-1"); got != "" { |
| 964 | t.Fatalf("startup, opt-out = %q, want none", got) |
| 965 | } |
| 966 | if mode, err := mintLayoutIntentWith(ctx, s, rows(nestedManaged)); err != nil || mode != "" { |
| 967 | t.Fatalf("mint, opt-out = %q %v, want none", mode, err) |
| 968 | } |
| 969 | // A recorded intent still wins over the opt-out. |
| 970 | appendIntent(t, s, release.LayoutModeNested, "") |
| 971 | if mode, err := mintLayoutIntent(ctx, s); err != nil || mode != release.LayoutModeNested { |
| 972 | t.Fatalf("mint under a recorded nested intent with the opt-out set = %q %v, want nested", mode, err) |
| 973 | } |
| 974 | } |
| 975 | |
| 976 | // newLayoutPass arms the default's gate with the same nestedOK the pass |
| 977 | // refuses nested work with. |
| 978 | func TestNewLayoutPassArmsTheDefaultGate(t *testing.T) { |
| 979 | resetLayoutGlobals(t) |
| 980 | want := errors.New("the pass's gate") |
| 981 | newLayoutPass(nil, nil, t.TempDir(), func() error { return want }, true) |
| 982 | gate := layoutDefaultGate.Load() |
| 983 | if gate == nil || !errors.Is((*gate)(), want) { |
| 984 | t.Fatal("the default's gate is not the pass's nestedOK") |
| 985 | } |
| 986 | } |
| 987 | |
| 988 | // newLayoutLeg kicks the leg once at build when a pass is armed, so the |
| 989 | // first evaluation with the gate's real answer never waits on a console |
| 990 | // bump; with no pass armed there is nothing to evaluate and no kick. |
| 991 | func TestNewLayoutLegKicksOnceWhenAPassIsArmed(t *testing.T) { |
| 992 | resetLayoutGlobals(t) |
| 993 | var runs atomic.Int32 |
| 994 | run := func(context.Context) bool { runs.Add(1); return false } |
| 995 | |
| 996 | leg := newLayoutLeg(context.Background(), run) |
| 997 | leg.wait() |
| 998 | if runs.Load() != 0 { |
| 999 | t.Fatalf("runs with no pass armed = %d, want 0", runs.Load()) |
| 1000 | } |
| 1001 | |
| 1002 | armedLayoutPass.Store(&layout.Pass{DataDir: t.TempDir(), SocketDir: t.TempDir()}) |
| 1003 | leg = newLayoutLeg(context.Background(), run) |
| 1004 | leg.wait() |
| 1005 | if runs.Load() != 1 { |
| 1006 | t.Fatalf("runs with a pass armed = %d, want exactly one startup kick", runs.Load()) |
| 1007 | } |
| 1008 | // Later kicks are the serial loop's, as before. |
| 1009 | leg.kick(context.Background()) |
| 1010 | leg.wait() |
| 1011 | if runs.Load() != 2 { |
| 1012 | t.Fatalf("runs after a serial kick = %d, want 2", runs.Load()) |
| 1013 | } |
| 1014 | } |
| 1015 | |
| 1016 | // A run under the nested default that the gate refuses asks for the leg's |
| 1017 | // retry, so a quiet fleet re-asks the gate on its own; the opt-out and a |
| 1018 | // typo do not, because nothing they wait on will change. |
| 1019 | func TestLayoutLegRetriesWhileTheDefaultIsBlocked(t *testing.T) { |
| 1020 | resetLayoutGlobals(t) |
| 1021 | armedLayoutPass.Store(&layout.Pass{DataDir: t.TempDir(), SocketDir: t.TempDir()}) |
| 1022 | prev := layoutBuild |
| 1023 | t.Cleanup(func() { layoutBuild = prev }) |
| 1024 | layoutBuild = func() string { return "v0.25.0" } |
| 1025 | s := release.NewDirStore(t.TempDir()) |
| 1026 | ctx := context.Background() |
| 1027 | converge := func(context.Context, layout.Mode, bool) []layout.Result { return nil } |
| 1028 | |
| 1029 | t.Setenv(layoutDefaultEnv, "") |
| 1030 | armDefaultGate(t, errors.New("not yet run")) |
| 1031 | if !layoutLegWith(ctx, s, "kass-1", converge) { |
| 1032 | t.Fatal("a gate that is not ready must ask for a retry") |
| 1033 | } |
| 1034 | armDefaultGate(t, nil) |
| 1035 | if layoutLegWith(ctx, s, "kass-1", converge) { |
| 1036 | t.Fatal("a ready box with nothing deferred must not ask for a retry") |
| 1037 | } |
| 1038 | t.Setenv(layoutDefaultEnv, "flat") |
| 1039 | if layoutLegWith(ctx, s, "kass-1", converge) { |
| 1040 | t.Fatal("the opt-out waits on nothing") |
| 1041 | } |
| 1042 | t.Setenv(layoutDefaultEnv, "nestd") |
| 1043 | if layoutLegWith(ctx, s, "kass-1", converge) { |
| 1044 | t.Fatal("a typo waits on nothing") |
| 1045 | } |
| 1046 | } |
| 1047 | |
amadan