| 1 | package host |
| 2 | |
| 3 | // The layout pass on the box (isolated-exec design §8.2): recovery at agent |
| 4 | // start, before the activator can wake anything, and convergence to the |
| 5 | // fleet's recorded layout intent on every serial move after that. The pass |
| 6 | // itself lives in internal/layout; this file is how the agent builds it, |
| 7 | // publishes it to the loops that run on other goroutines, and keeps it off |
| 8 | // the serial loop's own goroutine. |
| 9 | |
| 10 | import ( |
| 11 | "context" |
| 12 | "fmt" |
| 13 | "log" |
| 14 | "os" |
| 15 | "os/user" |
| 16 | "strconv" |
| 17 | "strings" |
| 18 | "sync" |
| 19 | "sync/atomic" |
| 20 | "time" |
| 21 | |
| 22 | "github.com/carlosframework/platform/internal/activator" |
| 23 | "github.com/carlosframework/platform/internal/layout" |
| 24 | "github.com/carlosframework/platform/internal/provision" |
| 25 | "github.com/carlosframework/platform/internal/registry" |
| 26 | "github.com/carlosframework/platform/internal/release" |
| 27 | "github.com/carlosframework/platform/internal/status" |
| 28 | "github.com/carlosframework/platform/internal/version" |
| 29 | ) |
| 30 | |
| 31 | // armedLayoutPass is the box's one layout pass. RunAgent stores it once, |
| 32 | // after startup recovery and before the activator starts, and the serial |
| 33 | // leg (fastConvergeLoop's goroutine) and the status snapshot read it. Nil |
| 34 | // means no activator is armed, so there is nothing for a pass to move. |
| 35 | var armedLayoutPass atomic.Pointer[layout.Pass] |
| 36 | |
| 37 | // layoutReload is the live edge's Reload, published by the edge's ready |
| 38 | // hook, which runs after the pass is built. Until then the pass's reload is |
| 39 | // a no-op and the edge's own 2s registry poll picks up a flipped row. |
| 40 | var layoutReload atomic.Pointer[func()] |
| 41 | |
| 42 | // layoutIntentSeen is the last outcome of a layout intent read for this box: |
| 43 | // "" when none is recorded or none addresses it, layout.IntentUnreadable |
| 44 | // when the last read failed or named a mode this binary does not know. A |
| 45 | // later successful read replaces whichever of the two it holds, so this |
| 46 | // only ever needs to keep "no intent" apart from "we could not tell". |
| 47 | var layoutIntentSeen atomic.Pointer[string] |
| 48 | |
| 49 | // lastLayoutIntent is the mode layoutIntentSeen holds, "" before any read. |
| 50 | func lastLayoutIntent() string { |
| 51 | if m := layoutIntentSeen.Load(); m != nil { |
| 52 | return *m |
| 53 | } |
| 54 | return "" |
| 55 | } |
| 56 | |
| 57 | func noteLayoutIntent(mode string) { |
| 58 | layoutIntentSeen.Store(&mode) |
| 59 | layoutDefaultSeen.Store(&layoutDefaultState{}) |
| 60 | } |
| 61 | |
| 62 | // layoutDefaultEnv is the deployment's answer when no layout intent is |
| 63 | // recorded (isolated-exec design §8.6, "Nested by default"): nested, or |
| 64 | // flat for a deployment that is not ready to isolate its tenants. Read |
| 65 | // from host.env like CARLOS_SYSTEM_CHANNEL, because it is a fact about the |
| 66 | // box's deployment that an operator sets once at bootstrap, and the agent |
| 67 | // reads every such fact the same way. |
| 68 | const layoutDefaultEnv = "CARLOS_LAYOUT_DEFAULT" |
| 69 | |
| 70 | // layoutDefault reads CARLOS_LAYOUT_DEFAULT: "" and "nested" are nested, |
| 71 | // "flat" is flat. Anything else is flat with the raw value returned as |
| 72 | // invalid, so the caller can log it: an operator who typed the variable |
| 73 | // meant to choose, and leaving the box as it was (flat) while saying so is |
| 74 | // the reading that cannot surprise them. |
| 75 | func layoutDefault(getenv func(string) string) (mode string, invalid string) { |
| 76 | v := strings.ToLower(strings.TrimSpace(getenv(layoutDefaultEnv))) |
| 77 | switch v { |
| 78 | case "", release.LayoutModeNested: |
| 79 | return release.LayoutModeNested, "" |
| 80 | case release.LayoutModeFlat: |
| 81 | return release.LayoutModeFlat, "" |
| 82 | } |
| 83 | return release.LayoutModeFlat, v |
| 84 | } |
| 85 | |
| 86 | // layoutDefaultGate is the readiness the default consults before it stands |
| 87 | // in for a nested intent: the same nestedGate the exec backend and the |
| 88 | // layout pass refuse nested work with (instancefs.SelfCheck, the three |
| 89 | // nosymfollow mounts, the control service's probe verdict). Stored when the |
| 90 | // pass is built; nil means no activator is armed, so nothing could wake |
| 91 | // nested and the default does not apply. |
| 92 | var layoutDefaultGate atomic.Pointer[func() error] |
| 93 | |
| 94 | // layoutDefaultState is what status says about the default: Applied when |
| 95 | // the box is nested by default rather than by a recorded intent, Blocked |
| 96 | // with the reason when the deployment's default is nested but this box is |
| 97 | // not ready for it. |
| 98 | type layoutDefaultState struct { |
| 99 | Applied bool |
| 100 | Blocked string |
| 101 | } |
| 102 | |
| 103 | var layoutDefaultSeen atomic.Pointer[layoutDefaultState] |
| 104 | |
| 105 | func lastLayoutDefault() layoutDefaultState { |
| 106 | if d := layoutDefaultSeen.Load(); d != nil { |
| 107 | return *d |
| 108 | } |
| 109 | return layoutDefaultState{} |
| 110 | } |
| 111 | |
| 112 | // layoutBuild is the running build the default's floor check places; |
| 113 | // a var so a test can name a release. |
| 114 | var layoutBuild = func() string { return version.Build } |
| 115 | |
| 116 | // defaultLayoutIntent is the effective intent when none is recorded |
| 117 | // (design §8.6, "Nested by default"). It is nested only on a box that is |
| 118 | // READY: the deployment's default is nested (CARLOS_LAYOUT_DEFAULT), the |
| 119 | // running build is at or above release.LayoutFloor when it can be placed |
| 120 | // (a build that evaluates this rule carries the pass by construction, so |
| 121 | // an unplaceable dev build passes; a mis-tagged release below the floor |
| 122 | // does not), and the nested gate answers: SelfCheck, the mounts and the |
| 123 | // probe verdict. Anything else is "" (flat, as before the default), with |
| 124 | // the reason recorded for status so an operator can see why a box has |
| 125 | // not moved on its own. The default never forces: an awake host gets its |
| 126 | // grace exactly as under a recorded intent. |
| 127 | func defaultLayoutIntent() (mode string, reason string) { |
| 128 | def, invalid := layoutDefault(os.Getenv) |
| 129 | if invalid != "" { |
| 130 | return "", layoutDefaultEnv + "=" + invalid + " is not nested or flat; staying flat" |
| 131 | } |
| 132 | if def != release.LayoutModeNested { |
| 133 | return "", "" |
| 134 | } |
| 135 | if b := layoutBuild(); release.BelowFloor(b, release.LayoutFloor) { |
| 136 | return "", "this box runs " + b + ", below v" + release.LayoutFloor.String() + ", the first release on which nested instances wake" |
| 137 | } |
| 138 | gate := layoutDefaultGate.Load() |
| 139 | if gate == nil { |
| 140 | return "", "no activator is armed on this box, so nothing could wake nested" |
| 141 | } |
| 142 | if err := (*gate)(); err != nil { |
| 143 | return "", "isolation not ready: " + err.Error() |
| 144 | } |
| 145 | return release.LayoutModeNested, "" |
| 146 | } |
| 147 | |
| 148 | // noteLayoutDefault records the default's outcome for status: nested by |
| 149 | // default when mode is nested, otherwise none with the reason it did not |
| 150 | // apply. |
| 151 | func noteLayoutDefault(mode, reason string) { |
| 152 | if mode == release.LayoutModeNested { |
| 153 | layoutIntentSeen.Store(&mode) |
| 154 | layoutDefaultSeen.Store(&layoutDefaultState{Applied: true}) |
| 155 | return |
| 156 | } |
| 157 | none := release.LayoutModeNone |
| 158 | layoutIntentSeen.Store(&none) |
| 159 | layoutDefaultSeen.Store(&layoutDefaultState{Blocked: reason}) |
| 160 | } |
| 161 | |
| 162 | // layoutStatusReport builds the box's layout report for the status |
| 163 | // snapshot (design §8.4, amendment R15): the mode this box last converged |
| 164 | // to, and every instance-kind route's word from the armed pass's own |
| 165 | // Classify — the same classification a wake would see, so status never |
| 166 | // says something different from what the activator actually does. |
| 167 | // |
| 168 | // With no activator armed there is no pass and no Classify, but the box |
| 169 | // is not silent either: it reports Mode "unmanaged" with every |
| 170 | // instance-kind row judged by path shape alone (provision.NestedPaths), |
| 171 | // because the one thing a reader needs to know about such a box is |
| 172 | // whether a downgrade would strand a nested database, and that is a fact |
| 173 | // about paths, not about wakes. Nothing wakes a hibernate-flagged row on |
| 174 | // such a box, so no row there is managed, and the console's downgrade |
| 175 | // guard (layoutguard.go) treats the report like any other: a row named |
| 176 | // nested blocks, flat rows do not. Silence from a box at or above the |
| 177 | // floor would otherwise block its fleet's rollback for good. |
| 178 | // |
| 179 | // routes is the same list the caller already read for its other probes; |
| 180 | // this never lists the registry itself, mirroring every other probe in |
| 181 | // this file. |
| 182 | func layoutStatusReport(routes []registry.Route) *status.LayoutReport { |
| 183 | pass := armedLayoutPass.Load() |
| 184 | if pass == nil { |
| 185 | return unmanagedLayoutReport(routes) |
| 186 | } |
| 187 | mode := lastLayoutIntent() |
| 188 | if mode == "" { |
| 189 | mode = "none" |
| 190 | } |
| 191 | def := lastLayoutDefault() |
| 192 | rows := map[string]status.RowLayout{} |
| 193 | for _, rt := range routes { |
| 194 | // Only instance-kind rows: a sidecar (Kind "sidecar") is outside |
| 195 | // the pass entirely (finding S6), and so is everything that is |
| 196 | // neither an instance nor a sidecar (static, service). |
| 197 | if rt.Kind != "instance" { |
| 198 | continue |
| 199 | } |
| 200 | switch { |
| 201 | case activator.ActivatorManaged(rt): |
| 202 | word, reason := pass.Classify(rt, routes) |
| 203 | // A flat host the pass keeps fenced and down after a refused |
| 204 | // nested move classifies flat (no journal), but it does not |
| 205 | // wake: status says so, loudly, with the refusal. |
| 206 | if why, held := pass.Held(rt.Host); held && word == activator.LayoutFlat { |
| 207 | word, reason = activator.LayoutFenced, "held down after a refused move to nested: "+why |
| 208 | } |
| 209 | rows[rt.Host] = status.RowLayout{Layout: string(word), Reason: reason, Note: pass.Note(rt.Host)} |
| 210 | case rt.Socket != "" && !rt.Hibernate: |
| 211 | // ActivatorManaged's own doc: an unflagged socket route is run |
| 212 | // by another supervisor, nobody's to migrate or isolate. |
| 213 | rows[rt.Host] = status.RowLayout{Layout: "external"} |
| 214 | } |
| 215 | } |
| 216 | return &status.LayoutReport{Mode: mode, Rows: rows, Default: def.Applied, DefaultBlocked: def.Blocked} |
| 217 | } |
| 218 | |
| 219 | // LayoutModeUnmanaged is the report Mode of a box with no activator armed: |
| 220 | // no pass runs there and no hibernate-flagged row ever wakes, so its rows |
| 221 | // are judged by path shape alone (see layoutStatusReport). |
| 222 | const LayoutModeUnmanaged = "unmanaged" |
| 223 | |
| 224 | // unmanagedLayoutReport is layoutStatusReport's answer with no pass armed. |
| 225 | // Instance-kind rows only, as the managed report: fenced when a journal or |
| 226 | // hold marker names the host (a move was in flight, or a refused move kept |
| 227 | // the host down, when the activator went away, and the files are still |
| 228 | // where recovery expects them), nested by path shape, external for an |
| 229 | // unflagged socket route, flat otherwise. The fenced check comes first, |
| 230 | // as in activator.Classify: a mid-move host's row can still carry flat |
| 231 | // paths while its files are already between two directories. |
| 232 | func unmanagedLayoutReport(routes []registry.Route) *status.LayoutReport { |
| 233 | rows := map[string]status.RowLayout{} |
| 234 | dd := provisionTickConfig(os.Getenv).DataDir |
| 235 | for _, rt := range routes { |
| 236 | if rt.Kind != "instance" { |
| 237 | continue |
| 238 | } |
| 239 | switch { |
| 240 | case layout.JournalPresent(dd, rt.Host): |
| 241 | rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutFenced), Reason: "layout migration in flight; see " + provision.LayoutJournalPath(dd, rt.Host) + " (no activator on this box to recover it)"} |
| 242 | case layout.HoldMarkerPresent(dd, rt.Host): |
| 243 | rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutFenced), Reason: "held down after a refused move to nested (no activator on this box to retry it)"} |
| 244 | case provision.NestedPaths(dd, rt): |
| 245 | rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutNested), Reason: "no activator on this box; judged by path shape"} |
| 246 | case rt.Socket != "" && !rt.Hibernate: |
| 247 | rows[rt.Host] = status.RowLayout{Layout: "external"} |
| 248 | case rt.Hibernate && rt.Socket != "": |
| 249 | rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutFlat)} |
| 250 | } |
| 251 | } |
| 252 | return &status.LayoutReport{Mode: LayoutModeUnmanaged, Rows: rows} |
| 253 | } |
| 254 | |
| 255 | // reloadRouting asks the live edge to rebuild its routes now, if there is |
| 256 | // one yet. |
| 257 | func reloadRouting() { |
| 258 | if f := layoutReload.Load(); f != nil { |
| 259 | (*f)() |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | // serializeWithConfigSync runs f under the lock every SyncExec pass holds, |
| 264 | // so config sync never renders a host's bundle into a directory the pass |
| 265 | // is moving the host out of. The pass holds nothing but its own fence when |
| 266 | // it asks, and SyncExec under this lock never waits on a fence or on the |
| 267 | // pass (its bounce only starts a stop), so the two cannot wait on each |
| 268 | // other. |
| 269 | func serializeWithConfigSync(f func()) { |
| 270 | configSyncMu.Lock() |
| 271 | defer configSyncMu.Unlock() |
| 272 | f() |
| 273 | } |
| 274 | |
| 275 | // socketsGID is the carlos-sockets group's id, which a nested host's run |
| 276 | // directory is given so the edge can reach its socket, or -1 to leave the |
| 277 | // group alone where the group does not exist (a dev machine). |
| 278 | func socketsGID() int { |
| 279 | g, err := user.LookupGroup(socketsGroup) |
| 280 | if err != nil { |
| 281 | return -1 |
| 282 | } |
| 283 | n, err := strconv.Atoi(g.Gid) |
| 284 | if err != nil { |
| 285 | return -1 |
| 286 | } |
| 287 | return n |
| 288 | } |
| 289 | |
| 290 | // nestedRowsPresent reports whether any activator-managed row on the box is |
| 291 | // already at the nested paths. |
| 292 | func nestedRowsPresent(rows []registry.Route) bool { |
| 293 | dd := provisionTickConfig(os.Getenv).DataDir |
| 294 | for _, rt := range rows { |
| 295 | if activator.ActivatorManaged(rt) && provision.NestedPaths(dd, rt) { |
| 296 | return true |
| 297 | } |
| 298 | } |
| 299 | return false |
| 300 | } |
| 301 | |
| 302 | // newLayoutPass builds the pass from what only RunAgent has in hand. |
| 303 | // nestedOK must be the same gate the exec backend refuses nested wakes |
| 304 | // with, so the pass never makes a nested tenant the box would refuse to |
| 305 | // start. swept is whether the startup unit sweep succeeded; without it the |
| 306 | // pass never trusts an unproven stop. |
| 307 | // |
| 308 | // DeferAwake is set because the only Converge caller is the serial leg, |
| 309 | // which runs while tenants serve; startup recovery never reaches the code |
| 310 | // that reads it. |
| 311 | func newLayoutPass(reg *registry.Registry, svc *activator.Service, dd string, nestedOK func() error, swept bool) *layout.Pass { |
| 312 | p := &layout.Pass{ |
| 313 | Registry: reg, |
| 314 | DataDir: dd, |
| 315 | SocketsGID: socketsGID(), |
| 316 | Reload: reloadRouting, |
| 317 | Logf: log.Printf, |
| 318 | SweptAtStart: swept, |
| 319 | NestedOK: nestedOK, |
| 320 | Serialize: serializeWithConfigSync, |
| 321 | DeferAwake: true, |
| 322 | } |
| 323 | // Assigned only when non-nil: a nil *Service inside the interface would |
| 324 | // not be a nil Fence, and the pass would call through it. |
| 325 | if svc != nil { |
| 326 | p.Fence = svc |
| 327 | } |
| 328 | // The pass fences from the box's generation source, the one the import |
| 329 | // rollback's journal stores its generation from (fenceGenerations), so |
| 330 | // a rollback resumed after a restart never meets this pass's fence |
| 331 | // under its own number. A source that fails falls back to the pass's |
| 332 | // own default, the wall clock. |
| 333 | if reg != nil { |
| 334 | next := fenceGenerations(reg) |
| 335 | p.Generation = func() uint64 { |
| 336 | if g, err := next(); err == nil { |
| 337 | return g |
| 338 | } |
| 339 | return uint64(time.Now().UnixNano()) |
| 340 | } |
| 341 | } |
| 342 | // The default's readiness is this same gate (defaultLayoutIntent). |
| 343 | if nestedOK != nil { |
| 344 | layoutDefaultGate.Store(&nestedOK) |
| 345 | } |
| 346 | return p |
| 347 | } |
| 348 | |
| 349 | // startActivator is the order the activator comes up in. The startup unit |
| 350 | // sweep first, so nothing an earlier agent left is running. Then layout |
| 351 | // recovery, which runs even when the sweep failed (completing a journal |
| 352 | // is exact) and is told whether it succeeded. Then Start, so no host is |
| 353 | // woken while a journal says its files are between two directories. |
| 354 | func startActivator(ctx context.Context, sweep func(context.Context) error, recoverLayout func(ctx context.Context, swept bool), start func()) { |
| 355 | err := sweep(ctx) |
| 356 | recoverLayout(ctx, err == nil) |
| 357 | start() |
| 358 | } |
| 359 | |
| 360 | // activatorStart keeps sweep, recovery and Start ordered. Legacy boxes run |
| 361 | // the sequence in the background; claim boxes still call Start alone. |
| 362 | // Ready releases requests and agent duties only after Start returns. The |
| 363 | // returned channel also joins the instance-key pass on shutdown. |
| 364 | type activatorStart struct { |
| 365 | Configure func(context.Context) |
| 366 | Ready func() |
| 367 | Claims bool // claims.Writes.Enabled: a claim box |
| 368 | Sweep func(context.Context) error // the instance-unit sweep |
| 369 | Recover func(context.Context, bool) *layout.Pass // builds the pass and recovers under it; the bool is whether the sweep succeeded |
| 370 | Routes func() ([]registry.Route, error) // the registry's List |
| 371 | Route func(string) (*registry.Route, error) // the registry's Get: each row is read again in its host's slot |
| 372 | Keys instanceKeyBackfiller // the exec backend |
| 373 | Slot hostPreparationSlot // the service's PrepareScheduledToken |
| 374 | Publish func(*layout.Pass) // armedLayoutPass.Store |
| 375 | Start func() // the service's Start |
| 376 | } |
| 377 | |
| 378 | // Run returns immediately on legacy boxes. Cancellation prevents a late |
| 379 | // Start and reaches the key pass through the agent lifetime context. |
| 380 | func (s activatorStart) Run(ctx context.Context) <-chan struct{} { |
| 381 | done := make(chan struct{}) |
| 382 | start := func() { |
| 383 | if ctx.Err() == nil { |
| 384 | s.Start() |
| 385 | if s.Ready != nil { |
| 386 | s.Ready() |
| 387 | } |
| 388 | } |
| 389 | } |
| 390 | if s.Claims { |
| 391 | if s.Configure != nil { |
| 392 | s.Configure(ctx) |
| 393 | } |
| 394 | start() |
| 395 | close(done) |
| 396 | return done |
| 397 | } |
| 398 | go func() { |
| 399 | defer close(done) |
| 400 | if s.Configure != nil { |
| 401 | s.Configure(ctx) |
| 402 | } |
| 403 | if ctx.Err() != nil { |
| 404 | return |
| 405 | } |
| 406 | startActivator(ctx, s.Sweep, func(ctx context.Context, swept bool) { |
| 407 | if ctx.Err() == nil { |
| 408 | s.Publish(s.Recover(ctx, swept)) |
| 409 | } |
| 410 | }, start) |
| 411 | if ctx.Err() != nil { |
| 412 | return |
| 413 | } |
| 414 | routes, err := s.Routes() |
| 415 | if err != nil { |
| 416 | log.Printf("agent: instance keys: list routes: %v (each host's next start mints its key)", err) |
| 417 | return |
| 418 | } |
| 419 | backfillInstanceKeys(ctx, routes, keyBackfillInSlot{keys: s.Keys, slot: s.Slot, route: s.Route}) |
| 420 | }() |
| 421 | return done |
| 422 | } |
| 423 | |
| 424 | // layoutIntentTimeout bounds the one intent read at startup, which the |
| 425 | // activator's start waits on. |
| 426 | const layoutIntentTimeout = 30 * time.Second |
| 427 | |
| 428 | // startupLayoutIntent reads the intent for startup recovery when there is |
| 429 | // a journal to recover (startupRecoveryIntent). It is "" |
| 430 | // when no intent is recorded or none addresses this box. It is |
| 431 | // IntentUnreadable when the box cannot be sure: no deploy store, a failed |
| 432 | // read, or a request whose mode this binary does not know. Under that |
| 433 | // intent recovery holds a completed revert fenced rather than release a |
| 434 | // host flat that the fleet may want nested. |
| 435 | func startupLayoutIntent(ctx context.Context, store release.Store, storeErr error, boxLabel string) layout.Mode { |
| 436 | if storeErr != nil { |
| 437 | log.Printf("layout: intent -> unreadable: deploy store: %v", storeErr) |
| 438 | noteLayoutIntent(string(layout.IntentUnreadable)) |
| 439 | return layout.IntentUnreadable |
| 440 | } |
| 441 | ctx, cancel := context.WithTimeout(ctx, layoutIntentTimeout) |
| 442 | defer cancel() |
| 443 | in, found, err := release.LoadLayoutIntent(ctx, store, boxLabel) |
| 444 | switch { |
| 445 | case err != nil: |
| 446 | log.Printf("layout: intent -> unreadable: %v", err) |
| 447 | noteLayoutIntent(string(layout.IntentUnreadable)) |
| 448 | return layout.IntentUnreadable |
| 449 | case !found: |
| 450 | // No recorded intent. The default is not asked its gate here: at |
| 451 | // agent start the control service's probe has not run yet (its |
| 452 | // /ready answers "not yet run" until the first async probe lands), |
| 453 | // so the gate would say "not ready" on every start, and recovery |
| 454 | // under "" drops every hold marker (RestoreHolds) and releases a |
| 455 | // completed revert, handing a tenant that kept its move refused a |
| 456 | // fresh grace to run flat. So when the deployment's default is |
| 457 | // nested, startup recovery runs under IntentUnreadable, the |
| 458 | // fail-closed intent: holds and journals are kept, nothing is |
| 459 | // released flat, and the first serial move reads the intent with |
| 460 | // the gate's real answer. The opt-out (CARLOS_LAYOUT_DEFAULT=flat) |
| 461 | // keeps today's behaviour, dropping holds as a recorded flat does. |
| 462 | if def, invalid := layoutDefault(os.Getenv); def == release.LayoutModeNested && invalid == "" { |
| 463 | log.Printf("layout: no intent recorded at start -> holds and journals kept until the first serial move asks the isolation gate (nested by default)") |
| 464 | noteLayoutDefault("", "agent starting: holds and journals kept until the first serial move asks the isolation gate") |
| 465 | return layout.IntentUnreadable |
| 466 | } |
| 467 | mode, reason := defaultLayoutIntent() |
| 468 | noteLayoutDefault(mode, reason) |
| 469 | return layout.Mode(mode) |
| 470 | case !release.ValidLayoutMode(in.Mode): |
| 471 | log.Printf("layout: intent %s -> unreadable: mode %q is not flat or nested", in.ID, in.Mode) |
| 472 | noteLayoutIntent(string(layout.IntentUnreadable)) |
| 473 | return layout.IntentUnreadable |
| 474 | } |
| 475 | noteLayoutIntent(in.Mode) |
| 476 | return layout.Mode(in.Mode) |
| 477 | } |
| 478 | |
| 479 | // startupRecoveryIntent is the intent startup recovery runs under. The |
| 480 | // intent only matters to a journal, so with none in <data>/layout it is "" |
| 481 | // and the deploy store is not opened: the activator's start does not wait |
| 482 | // on a read it has no use for, and the published intent is left for the |
| 483 | // first serial leg to set. A layout directory that cannot be listed may |
| 484 | // hold journals, so it is read as usual. |
| 485 | func startupRecoveryIntent(ctx context.Context, dataDir string, open func(context.Context) (release.Store, error), boxLabel string) layout.Mode { |
| 486 | hosts, err := layout.JournalHosts(dataDir) |
| 487 | // A hold marker is the other thing startup acts on (RestoreHolds), so |
| 488 | // it too makes the intent worth reading. |
| 489 | holds, herr := layout.HoldMarkers(dataDir) |
| 490 | if err == nil && herr == nil && len(hosts) == 0 && len(holds) == 0 { |
| 491 | return "" |
| 492 | } |
| 493 | if herr != nil { |
| 494 | log.Printf("layout: list hold markers at start: %v; reading the intent anyway", herr) |
| 495 | } |
| 496 | if err != nil { |
| 497 | log.Printf("layout: list journals at start: %v; reading the intent anyway", err) |
| 498 | } |
| 499 | ctx, cancel := context.WithTimeout(ctx, layoutIntentTimeout) |
| 500 | defer cancel() |
| 501 | store, err := open(ctx) |
| 502 | return startupLayoutIntent(ctx, store, err, boxLabel) |
| 503 | } |
| 504 | |
| 505 | // recoverLayoutAtStart runs the pass's recovery under the startup intent, |
| 506 | // then restores the holds whose markers survived the restart, and logs |
| 507 | // what it did. Both run before the activator can wake anything. |
| 508 | func recoverLayoutAtStart(ctx context.Context, pass *layout.Pass, intent layout.Mode) { |
| 509 | logLayoutResults(pass.Recover(ctx, intent)) |
| 510 | logLayoutResults(pass.RestoreHolds(ctx, intent)) |
| 511 | } |
| 512 | |
| 513 | // layoutLeg runs the layout pass off the serial loop. A pass can wait up to |
| 514 | // a minute per host for a stop, and the restart, wake, schedule and system |
| 515 | // legs and the fast sweep must not wait behind it. One pass runs at a time; |
| 516 | // a serial move that lands while one is running marks the leg dirty, and |
| 517 | // the leg runs exactly once more after it, however many moves landed. |
| 518 | type layoutLeg struct { |
| 519 | run func(context.Context) |
| 520 | running atomic.Bool |
| 521 | dirty atomic.Bool |
| 522 | wg sync.WaitGroup |
| 523 | |
| 524 | retryMu sync.Mutex |
| 525 | retryPending bool |
| 526 | } |
| 527 | |
| 528 | // newLayoutLeg builds the leg around one run and, when a pass is already |
| 529 | // armed, kicks it once. The leg otherwise runs only on a serial move or |
| 530 | // its own retry timer, and startup recovery runs fail-closed without |
| 531 | // asking the isolation gate (startupLayoutIntent), so a box that restarts |
| 532 | // and then sees no console bump would keep what startup fenced for as |
| 533 | // long as the fleet stayed quiet. The kick is the first evaluation with |
| 534 | // the gate's real answer, and it depends on nothing but this process. |
| 535 | // fastConvergeLoop builds the leg after startActivator has returned, so |
| 536 | // the pass is armed by then on every box that has an activator; with none |
| 537 | // armed the kick reads nothing (runLayoutLeg). |
| 538 | func newLayoutLeg(ctx context.Context, run func(context.Context) bool) *layoutLeg { |
| 539 | l := &layoutLeg{} |
| 540 | l.run = func(ctx context.Context) { |
| 541 | if run(ctx) { |
| 542 | l.retryLater(ctx, awakeRetry) |
| 543 | } |
| 544 | } |
| 545 | if armedLayoutPass.Load() != nil { |
| 546 | l.kick(ctx) |
| 547 | } |
| 548 | return l |
| 549 | } |
| 550 | |
| 551 | // kick asks for a pass and returns at once. |
| 552 | func (l *layoutLeg) kick(ctx context.Context) { |
| 553 | l.dirty.Store(true) |
| 554 | if !l.running.CompareAndSwap(false, true) { |
| 555 | return |
| 556 | } |
| 557 | l.wg.Add(1) |
| 558 | go func() { |
| 559 | defer l.wg.Done() |
| 560 | for { |
| 561 | for l.dirty.Swap(false) { |
| 562 | l.run(ctx) |
| 563 | } |
| 564 | if !l.goIdle() { |
| 565 | return |
| 566 | } |
| 567 | } |
| 568 | }() |
| 569 | } |
| 570 | |
| 571 | // goIdle marks the runner idle and reports whether it must run again. A |
| 572 | // kick that landed after the runner's last look at dirty saw the leg |
| 573 | // running and left its request there; the runner takes it back up, unless |
| 574 | // that kick has already started a runner of its own. |
| 575 | func (l *layoutLeg) goIdle() bool { |
| 576 | l.running.Store(false) |
| 577 | return l.dirty.Load() && l.running.CompareAndSwap(false, true) |
| 578 | } |
| 579 | |
| 580 | // wait blocks until no pass is running or pending. |
| 581 | func (l *layoutLeg) wait() { l.wg.Wait() } |
| 582 | |
| 583 | // runLayoutLeg is one run of the serial leg: the published pass, converged |
| 584 | // to this box's intent. With no pass armed there is nothing to move, and |
| 585 | // the intent is not read. |
| 586 | func runLayoutLeg(ctx context.Context, store release.Store, boxLabel string) (retry bool) { |
| 587 | pass := armedLayoutPass.Load() |
| 588 | if pass == nil { |
| 589 | return false |
| 590 | } |
| 591 | return layoutLegWith(ctx, store, boxLabel, func(ctx context.Context, mode layout.Mode, force bool) []layout.Result { |
| 592 | // The pass is single-flight (the leg), so flipping its deferral |
| 593 | // for one run races nothing. A forced intent stops awake hosts |
| 594 | // now; an ordinary one gives them the pass's awake grace. |
| 595 | pass.DeferAwake = !force |
| 596 | return pass.Converge(ctx, mode) |
| 597 | }) |
| 598 | } |
| 599 | |
| 600 | // awakeRetry is how long after a run that deferred an awake host the leg |
| 601 | // runs again on its own. The leg otherwise runs only on a serial move, |
| 602 | // and the awake grace is a bound in wall time: without this a quiet |
| 603 | // fleet would never come back to stop the host once its grace ran out. |
| 604 | const awakeRetry = 10 * time.Minute |
| 605 | |
| 606 | // retryLater arms one timer to kick the leg again. A timer already |
| 607 | // pending is enough; the run it starts re-evaluates every host. |
| 608 | func (l *layoutLeg) retryLater(ctx context.Context, after time.Duration) { |
| 609 | l.retryMu.Lock() |
| 610 | defer l.retryMu.Unlock() |
| 611 | if l.retryPending { |
| 612 | return |
| 613 | } |
| 614 | l.retryPending = true |
| 615 | time.AfterFunc(after, func() { |
| 616 | l.retryMu.Lock() |
| 617 | l.retryPending = false |
| 618 | l.retryMu.Unlock() |
| 619 | if ctx.Err() == nil { |
| 620 | l.kick(ctx) |
| 621 | } |
| 622 | }) |
| 623 | } |
| 624 | |
| 625 | // layoutLegWith is the leg's decision: read the intent, and converge only |
| 626 | // when a request addresses this box with a mode this binary knows. An |
| 627 | // intent that cannot be read moves nothing and publishes |
| 628 | // layout.IntentUnreadable, overwriting whatever mode was last seen, since a |
| 629 | // read that just failed is not proof the fleet still wants it; the next |
| 630 | // serial move reads it again. |
| 631 | // |
| 632 | // It reports whether the run deferred an awake host, so the caller can |
| 633 | // come back for it when its grace runs out. |
| 634 | func layoutLegWith(ctx context.Context, store release.Store, boxLabel string, converge func(context.Context, layout.Mode, bool) []layout.Result) (retry bool) { |
| 635 | in, found, err := release.LoadLayoutIntent(ctx, store, boxLabel) |
| 636 | switch { |
| 637 | case err != nil: |
| 638 | log.Printf("layout: intent -> unreadable: %v; not converging this move", err) |
| 639 | noteLayoutIntent(string(layout.IntentUnreadable)) |
| 640 | return false |
| 641 | case !found: |
| 642 | // No recorded intent: the deployment's default stands in, on a |
| 643 | // box that is ready for it (design §8.6, "Nested by default"). It |
| 644 | // never forces, so an awake host keeps its grace; a box that is |
| 645 | // not ready converges nothing and status says why. Logged once |
| 646 | // per change of answer, not per serial move. |
| 647 | mode, reason := defaultLayoutIntent() |
| 648 | if prev := lastLayoutDefault(); prev.Blocked != reason || prev.Applied != (mode == release.LayoutModeNested) { |
| 649 | if mode == release.LayoutModeNested { |
| 650 | log.Printf("layout: no intent recorded -> nested by default (%s unset or nested; record `carlos system layout revert` to opt out)", layoutDefaultEnv) |
| 651 | } else if reason != "" { |
| 652 | log.Printf("layout: no intent recorded; nested by default is not in force -> %s", reason) |
| 653 | } |
| 654 | } |
| 655 | noteLayoutDefault(mode, reason) |
| 656 | if mode != release.LayoutModeNested { |
| 657 | // A box the default has not reached yet is retried on the |
| 658 | // leg's own timer (awakeRetry), for the same reason an awake |
| 659 | // host is: the probe that gates it answers in wall time (its |
| 660 | // first run lands seconds after the control service starts), |
| 661 | // and a quiet fleet would otherwise never come back to ask. |
| 662 | // The opt-out and a typo are not waiting on anything. |
| 663 | def, invalid := layoutDefault(os.Getenv) |
| 664 | return def == release.LayoutModeNested && invalid == "" |
| 665 | } |
| 666 | return logLayoutResults(converge(ctx, layout.ModeNested, false)) |
| 667 | case !release.ValidLayoutMode(in.Mode): |
| 668 | log.Printf("layout: intent %s -> unreadable: mode %q is not flat or nested; not converging this move", in.ID, in.Mode) |
| 669 | noteLayoutIntent(string(layout.IntentUnreadable)) |
| 670 | return false |
| 671 | } |
| 672 | noteLayoutIntent(in.Mode) |
| 673 | return logLayoutResults(converge(ctx, layout.Mode(in.Mode), in.Force)) |
| 674 | } |
| 675 | |
| 676 | // logLayoutResults logs every result that changed or withheld something, |
| 677 | // and reports whether any host was deferred for being awake. Those hosts |
| 678 | // are one line together: under a standing intent every serial move meets |
| 679 | // them until they sleep or their grace runs out. |
| 680 | // |
| 681 | // It reports whether the leg should run again on its own: a host deferred |
| 682 | // for being awake has a grace that runs out in wall time, and a held host |
| 683 | // should come back within awakeRetry of an operator clearing its reason, |
| 684 | // quiet fleet or not. |
| 685 | func logLayoutResults(rs []layout.Result) (retry bool) { |
| 686 | var awake []string |
| 687 | for _, r := range rs { |
| 688 | switch { |
| 689 | case r.Outcome == "unchanged": |
| 690 | case r.Outcome == "held": |
| 691 | retry = true |
| 692 | log.Printf("layout: %s: %s -> %s", r.Host, r.Outcome, r.Reason) |
| 693 | case r.Outcome == "deferred" && strings.HasPrefix(r.Reason, "awake"): |
| 694 | awake = append(awake, r.Host+" ("+r.Reason+")") |
| 695 | case r.Host == "": |
| 696 | log.Printf("layout: %s -> %s", r.Outcome, r.Reason) |
| 697 | case r.Reason == "": |
| 698 | log.Printf("layout: %s: %s", r.Host, r.Outcome) |
| 699 | default: |
| 700 | log.Printf("layout: %s: %s -> %s", r.Host, r.Outcome, r.Reason) |
| 701 | } |
| 702 | } |
| 703 | if len(awake) > 0 { |
| 704 | log.Printf("layout: deferred until asleep -> %s", strings.Join(awake, ", ")) |
| 705 | } |
| 706 | return retry || len(awake) > 0 |
| 707 | } |
| 708 | |
| 709 | // mintLayoutIntent is provision.ReconcileConfig.Layout for this box: the |
| 710 | // mode of the newest intent addressing its label, "" when none does. An |
| 711 | // unreadable intent is an error, and the reconciler defers its mints |
| 712 | // rather than minting flat on a box the fleet may want nested. The same |
| 713 | // read the layout leg makes, so a mint and a pass agree. |
| 714 | func mintLayoutIntent(ctx context.Context, store release.Store) (string, error) { |
| 715 | return mintLayoutIntentWith(ctx, store, nil) |
| 716 | } |
| 717 | |
| 718 | // mintLayoutIntentWith is mintLayoutIntent with the box's own rows in hand |
| 719 | // (nil: no registry to ask). With no intent recorded and the deployment's |
| 720 | // default nested, the default's gate is asked; and when the gate has not |
| 721 | // yet answered (the first reconcile after an agent restart runs before the |
| 722 | // control service's first probe) a box whose registry already holds an |
| 723 | // activator-managed row at the nested paths counts as nested for mints |
| 724 | // all the same: the box has evidently moved, and a flat row minted beside |
| 725 | // nested siblings is the exposure the default exists to end. The gate |
| 726 | // still guards every wake, so a nested row minted this way waits for it. |
| 727 | func mintLayoutIntentWith(ctx context.Context, store release.Store, routes func() ([]registry.Route, error)) (string, error) { |
| 728 | in, found, err := release.LoadLayoutIntent(ctx, store, BoxLabel()) |
| 729 | switch { |
| 730 | case err != nil: |
| 731 | return "", err |
| 732 | case !found: |
| 733 | mode, _ := defaultLayoutIntent() |
| 734 | if mode == release.LayoutModeNested { |
| 735 | return mode, nil |
| 736 | } |
| 737 | if def, invalid := layoutDefault(os.Getenv); def == release.LayoutModeNested && invalid == "" && routes != nil { |
| 738 | rows, err := routes() |
| 739 | if err != nil { |
| 740 | return "", fmt.Errorf("layout default: list routes: %w", err) |
| 741 | } |
| 742 | if nestedRowsPresent(rows) { |
| 743 | return release.LayoutModeNested, nil |
| 744 | } |
| 745 | } |
| 746 | return release.LayoutModeNone, nil |
| 747 | case !release.ValidLayoutMode(in.Mode): |
| 748 | return "", fmt.Errorf("layout intent %s names mode %q, which this binary does not know", in.ID, in.Mode) |
| 749 | } |
| 750 | return in.Mode, nil |
| 751 | } |
| 752 | |
amadan