| 1 | // The box half of a NARROWED restart (discussion #16): the same two-process |
| 2 | // dance restart.go describes, reading the request log at |
| 3 | // release.RestartScopeKey instead of the app-wide generation, and cycling only |
| 4 | // the routes a request actually names. |
| 5 | // |
| 6 | // Everything load-bearing in restart.go still holds here, and this file is |
| 7 | // deliberately built to inherit rather than restate it: |
| 8 | // |
| 9 | // - It rides the SERIAL. No poll of its own; the agent's existing 2s serial |
| 10 | // read is what makes either object get looked at, and an idle box reads |
| 11 | // neither. The cost of this feature at rest is zero, and on a serial MOVE |
| 12 | // it is one extra GET per app on this box. |
| 13 | // - SEED-DON'T-REPLAY, per app, on this object's own mark. A box that has |
| 14 | // never looked records where the log stands and cycles nothing. |
| 15 | // - HIBERNATING routes are never woken. The root half skips them outright; |
| 16 | // the agent's bounce is a no-op on a sleeping instance and cycles a warm |
| 17 | // one. That is restart.go's rule verbatim, and narrowing the route set |
| 18 | // cannot weaken it. |
| 19 | // - A selector this binary does not understand addresses NOTHING. The |
| 20 | // request is stepped over and the mark advances past it, so a future |
| 21 | // `--region` cannot make today's box cycle an app. |
| 22 | // |
| 23 | // The one thing that is genuinely different: the CONTENT is read, not just the |
| 24 | // etag. See internal/release/restartscope.go for why that grants nothing — |
| 25 | // the account, the app and the candidate routes all come from this box's own |
| 26 | // registry, and a selector can only ever pick a subset of them. |
| 27 | package host |
| 28 | |
| 29 | import ( |
| 30 | "context" |
| 31 | "crypto/ed25519" |
| 32 | "fmt" |
| 33 | "log" |
| 34 | "sort" |
| 35 | "time" |
| 36 | |
| 37 | "github.com/carlosframework/platform/internal/registry" |
| 38 | "github.com/carlosframework/platform/internal/release" |
| 39 | ) |
| 40 | |
| 41 | // restartMemory is the agent's in-process memory for BOTH restart signals: the |
| 42 | // app-wide generation's etag, and the id of the last narrowed request acted |
| 43 | // on. One struct rather than two loose maps because they are seeded together, |
| 44 | // compared together on the same serial move, and are only ever meaningful as a |
| 45 | // pair — a leg holding one and not the other would replay or drop half a |
| 46 | // request set. |
| 47 | // |
| 48 | // It lives for the life of the agent process (fastConvergeLoop owns it, |
| 49 | // restartSeed fills it), which is the whole point: there is nowhere else to |
| 50 | // remember what "changed" means between two serial moves that may be days |
| 51 | // apart. |
| 52 | type restartMemory struct { |
| 53 | gen map[[2]string]string // app -> restart-gen.json etag, or genAbsent |
| 54 | scope map[[2]string]string // app -> last applied request id, or release.RestartScopeNone |
| 55 | // retries is app -> the ids of the host-retry log's entries this |
| 56 | // process has handled (hostRetryPass). It is not a seed mark: an empty |
| 57 | // memory, at agent start, handles every entry in the log. |
| 58 | retries map[[2]string]map[string]bool |
| 59 | |
| 60 | // retryHost clears one host's failure hold, for a RestartScopeRetry |
| 61 | // request whose root-signed grant verified: the grant's build, and the |
| 62 | // restriction's book and epoch it was issued for (rolloutRetryHost; |
| 63 | // final review round 1, finding 3). It is the |
| 64 | // rollout manager's (rollout.Manager.RetryHost) when armed, and a |
| 65 | // declined box's that honours its failure book (failureHolds.retryHost, |
| 66 | // Task 10). nil otherwise: there is no hold to clear, and the request is |
| 67 | // dropped. |
| 68 | retryHost func(host, build, book string, epoch uint64) bool |
| 69 | // pub is the deployment root's public key (CARLOS_RELEASE_PUBKEY), which |
| 70 | // every retry's grant is verified against (spec §11). Nil drops every |
| 71 | // retry: a box that cannot tell a grant from a forgery clears nothing. |
| 72 | pub ed25519.PublicKey |
| 73 | } |
| 74 | |
| 75 | func newRestartMemory() *restartMemory { |
| 76 | return &restartMemory{gen: map[[2]string]string{}, scope: map[[2]string]string{}, retries: map[[2]string]map[string]bool{}} |
| 77 | } |
| 78 | |
| 79 | // scopedRestart is one app's outstanding narrowed work: the requests this box |
| 80 | // has not acted on yet, newest last. |
| 81 | type scopedRestart struct { |
| 82 | app [2]string |
| 83 | requests []release.RestartRequest |
| 84 | // history marks retries handed over because the box first saw the app or |
| 85 | // lost its place in the log, not because somebody asked for them since |
| 86 | // the last look (restartScopePass). A box with nothing that could clear |
| 87 | // a hold reads them as history, silently, as it always has |
| 88 | // (withoutHistory). |
| 89 | history bool |
| 90 | } |
| 91 | |
| 92 | // restartScopePass is the agent's comparison for narrowed requests — the twin |
| 93 | // of restartGenPass, with the same first-sight, absent-forever and |
| 94 | // read-error rules and for the same reasons. What it reports is not "this app |
| 95 | // moved" but WHICH REQUESTS are outstanding, because that is the only thing |
| 96 | // that says which routes to touch. |
| 97 | // |
| 98 | // Two requests inside one poll window are two entries in the log, so the pair |
| 99 | // is not collapsed into whichever landed last — the failure a bare generation |
| 100 | // object would have here, and the reason the object is a log at all. |
| 101 | // |
| 102 | // A mark that is no longer IN the log (this box was away long enough for its |
| 103 | // position to roll off) re-baselines and asks for nothing, with a line saying |
| 104 | // so. Restart requests are point-in-time events: a dropped one is recovered by |
| 105 | // typing the verb again, while a replayed one cycles production for something |
| 106 | // somebody asked for last week. |
| 107 | // |
| 108 | // A retry an earlier console left in this log is the one exception to both |
| 109 | // (Astra plan 9): it is not a restart, and a grant is not seed-skipped. On |
| 110 | // a first sight and on a lost position the log's retry entries are |
| 111 | // reported anyway, for the caller to verify and apply exactly as on a move |
| 112 | // (retryRequests, applyRetries): a grant spent or made stale since is a |
| 113 | // no-op under the failure book's generation rule, and an unsigned one is |
| 114 | // dropped with a line. withoutRetries keeps them out of every restart. |
| 115 | func restartScopePass(ctx context.Context, routes []registry.Route, store release.Store, cache map[[2]string]string) (out []scopedRestart, readErrs []error) { |
| 116 | apps := restartApps(routes) |
| 117 | live := make(map[[2]string]bool, len(apps)) |
| 118 | for _, k := range apps { |
| 119 | live[k] = true |
| 120 | g, _, err := release.LoadRestartScopes(ctx, store, k[0], k[1]) |
| 121 | if err != nil { |
| 122 | // Unreadable OR unparseable, one answer: leave the mark exactly as |
| 123 | // it was and evaluate the rest. Clearing it would turn a recovered |
| 124 | // blip into a bogus first sight; advancing it past a log this |
| 125 | // binary cannot read would drop the requests in it for good. |
| 126 | readErrs = append(readErrs, fmt.Errorf("%s/%s: %w", k[0], k[1], err)) |
| 127 | continue |
| 128 | } |
| 129 | newest := g.Newest() |
| 130 | prev, seen := cache[k] |
| 131 | if !seen { |
| 132 | cache[k] = newest // first sight: record, cycle nothing |
| 133 | out = appendHistory(out, k, g.Requests) |
| 134 | continue |
| 135 | } |
| 136 | if prev == newest { |
| 137 | continue |
| 138 | } |
| 139 | pending, found := g.PendingSince(prev) |
| 140 | cache[k] = newest |
| 141 | if !found { |
| 142 | log.Printf("agent: narrowed restarts for %s/%s: this box's position is no longer in the log — re-baselined, nothing cycled (run the verb again if something is still waiting)", k[0], k[1]) |
| 143 | out = appendHistory(out, k, g.Requests) |
| 144 | continue |
| 145 | } |
| 146 | if len(pending) > 0 { |
| 147 | out = append(out, scopedRestart{app: k, requests: pending}) |
| 148 | } |
| 149 | } |
| 150 | for k := range cache { |
| 151 | if !live[k] { |
| 152 | delete(cache, k) |
| 153 | } |
| 154 | } |
| 155 | return out, readErrs |
| 156 | } |
| 157 | |
| 158 | // scopedHosts resolves outstanding requests against this box's routes: which |
| 159 | // hosts of that app the requests actually name, deduplicated. |
| 160 | // |
| 161 | // Deduplicated across the whole set, not per request, because two pending |
| 162 | // requests that both name a host mean one cycle of it, not two — and a verb |
| 163 | // whose entire job is stopping processes must not manufacture a second stop |
| 164 | // out of an operator typing the same thing twice. |
| 165 | // |
| 166 | // An UNKNOWN selector kind is stepped over with a log line and addresses |
| 167 | // nothing. That is the forward-compatibility rule this feature is built on: |
| 168 | // a box that cannot understand a narrowing must never fall back to the wider |
| 169 | // thing. |
| 170 | // |
| 171 | // keep is the caller's own route filter — the root pass and the agent reach |
| 172 | // different processes and each passes its own — so this function decides |
| 173 | // membership and nothing else. |
| 174 | func scopedHosts(routes []registry.Route, app [2]string, requests []release.RestartRequest, keep func(registry.Route) bool) []string { |
| 175 | seen := map[string]bool{} |
| 176 | var out []string |
| 177 | for _, req := range requests { |
| 178 | // A retry is not a restart: it is handled by retryRequests, and must |
| 179 | // neither cycle anything nor be logged as a selector this build does |
| 180 | // not understand. |
| 181 | if req.Scope.Kind == release.RestartScopeRetry { |
| 182 | continue |
| 183 | } |
| 184 | // Asked of the KIND alone, once, before any route is considered: an |
| 185 | // app with no routes on this box must not look like an app whose |
| 186 | // selector we failed to understand. |
| 187 | if _, understood := req.Scope.Selects(registry.Route{}); !understood { |
| 188 | log.Printf("restart: narrowed request for %s/%s names selector %q, which this build does not understand — nothing cycled for it (upgrade carlos on this box)", |
| 189 | app[0], app[1], req.Scope.Kind) |
| 190 | continue |
| 191 | } |
| 192 | for _, rt := range routes { |
| 193 | if !release.Adoptable(rt) || rt.App != app[1] || release.RouteAccount(rt) != app[0] { |
| 194 | continue |
| 195 | } |
| 196 | if sel, _ := req.Scope.Selects(rt); !sel || seen[rt.Host] || !keep(rt) { |
| 197 | continue |
| 198 | } |
| 199 | seen[rt.Host] = true |
| 200 | out = append(out, rt.Host) |
| 201 | } |
| 202 | } |
| 203 | sort.Strings(out) |
| 204 | return out |
| 205 | } |
| 206 | |
| 207 | // retryRequest is one outstanding retry entry naming a host this box |
| 208 | // serves: the entry's account and app (the log's own), its host, and the |
| 209 | // grant it carries, unverified. |
| 210 | type retryRequest struct { |
| 211 | acct, app, host string |
| 212 | signed string |
| 213 | } |
| 214 | |
| 215 | // retryRequests is the retry entries of work, from the host-retry log |
| 216 | // (hostRetryPass) or a legacy retry in the narrowed log (restartScopePass): |
| 217 | // the entries, oldest first, whose host is one of this box's routes in the |
| 218 | // requesting app. A host on another box names nothing here. Entries are |
| 219 | // not merged by host: each carries its own grant, and the box verifies |
| 220 | // each (a forged entry must not hide a real one behind it). A grant spent |
| 221 | // by an earlier entry moves its restriction's epoch, so a second one for |
| 222 | // the same failure clears nothing. |
| 223 | func retryRequests(routes []registry.Route, work []scopedRestart) []retryRequest { |
| 224 | var out []retryRequest |
| 225 | for _, w := range work { |
| 226 | for _, req := range w.requests { |
| 227 | if req.Scope.Kind != release.RestartScopeRetry { |
| 228 | continue |
| 229 | } |
| 230 | for _, rt := range routes { |
| 231 | if rt.Host == req.Scope.Value && rt.App == w.app[1] && release.RouteAccount(rt) == w.app[0] { |
| 232 | out = append(out, retryRequest{acct: w.app[0], app: w.app[1], host: rt.Host, signed: req.Scope.Signed}) |
| 233 | break |
| 234 | } |
| 235 | } |
| 236 | } |
| 237 | } |
| 238 | return out |
| 239 | } |
| 240 | |
| 241 | // appendRetries adds app's retry entries among requests to out, as one |
| 242 | // scopedRestart, when there are any. |
| 243 | func appendRetries(out []scopedRestart, app [2]string, requests []release.RestartRequest) []scopedRestart { |
| 244 | var rs []release.RestartRequest |
| 245 | for _, req := range requests { |
| 246 | if req.Scope.Kind == release.RestartScopeRetry { |
| 247 | rs = append(rs, req) |
| 248 | } |
| 249 | } |
| 250 | if len(rs) == 0 { |
| 251 | return out |
| 252 | } |
| 253 | return append(out, scopedRestart{app: app, requests: rs}) |
| 254 | } |
| 255 | |
| 256 | // appendHistory is appendRetries for retries that are history: the box first |
| 257 | // saw the app, or lost its place in the log. |
| 258 | func appendHistory(out []scopedRestart, app [2]string, requests []release.RestartRequest) []scopedRestart { |
| 259 | n := len(out) |
| 260 | out = appendRetries(out, app, requests) |
| 261 | if len(out) > n { |
| 262 | out[n].history = true |
| 263 | } |
| 264 | return out |
| 265 | } |
| 266 | |
| 267 | // withoutHistory drops the retries that are history (scopedRestart.history) |
| 268 | // when this box has nothing that could clear a hold (retryHost nil), so the |
| 269 | // flag-off box does not log, for every legacy retry in a log it has just |
| 270 | // met, that its rollout manager is not armed. A retry somebody asked for |
| 271 | // since the last look is not history and still gets its line. |
| 272 | func withoutHistory(work []scopedRestart, mem *restartMemory) []scopedRestart { |
| 273 | if mem.retryHost != nil { |
| 274 | return work |
| 275 | } |
| 276 | var out []scopedRestart |
| 277 | for _, w := range work { |
| 278 | if !w.history { |
| 279 | out = append(out, w) |
| 280 | } |
| 281 | } |
| 282 | return out |
| 283 | } |
| 284 | |
| 285 | // hostRetryPass reads each app's host-retry log (release.HostRetryKey) and |
| 286 | // returns the entries this process has not handled yet, for retryRequests |
| 287 | // and applyRetries. It rides the serial like restartScopePass, one GET per |
| 288 | // app on a move, and none at all on a box with nothing that could clear a |
| 289 | // hold (retryHost nil), which no grant could change while the agent runs. |
| 290 | // |
| 291 | // It is not seed-skipped (Astra plan 9): a memory with no entry for an app, |
| 292 | // at agent start or for an app this box has just gained, hands over every |
| 293 | // entry in the log, so a grant written before an agent restart is applied |
| 294 | // after it. That is safe because a grant is checked, not trusted: one past |
| 295 | // its window, unsigned, or forged is dropped with a line, and one spent or |
| 296 | // made stale since is a no-op under the failure book's generation rule |
| 297 | // (RetryHostAt). Each entry is handled once per process: cache is the ids |
| 298 | // handled, rebuilt from the log on each read so it stays as small as the |
| 299 | // log, and an unchanged log hands over nothing, so nothing is verified |
| 300 | // twice. A read or parse error leaves the app's ids as they were. |
| 301 | func hostRetryPass(ctx context.Context, routes []registry.Route, store release.Store, mem *restartMemory) (out []scopedRestart, readErrs []error) { |
| 302 | if mem.retryHost == nil { |
| 303 | return nil, nil |
| 304 | } |
| 305 | apps := restartApps(routes) |
| 306 | live := make(map[[2]string]bool, len(apps)) |
| 307 | for _, k := range apps { |
| 308 | live[k] = true |
| 309 | g, _, err := release.LoadHostRetries(ctx, store, k[0], k[1]) |
| 310 | if err != nil { |
| 311 | readErrs = append(readErrs, fmt.Errorf("%s/%s host retries: %w", k[0], k[1], err)) |
| 312 | continue |
| 313 | } |
| 314 | done := mem.retries[k] |
| 315 | next := make(map[string]bool, len(g.Requests)) |
| 316 | var pending []release.RestartRequest |
| 317 | for _, req := range g.Requests { |
| 318 | if !done[req.ID] && !next[req.ID] { |
| 319 | pending = append(pending, req) |
| 320 | } |
| 321 | next[req.ID] = true |
| 322 | } |
| 323 | mem.retries[k] = next |
| 324 | out = appendRetries(out, k, pending) |
| 325 | } |
| 326 | for k := range mem.retries { |
| 327 | if !live[k] { |
| 328 | delete(mem.retries, k) |
| 329 | } |
| 330 | } |
| 331 | return out, readErrs |
| 332 | } |
| 333 | |
| 334 | // applyRetries verifies and applies each retry in work naming a host this |
| 335 | // box serves (retryRequests), the agent's one path for a grant from either |
| 336 | // log. A retry clears a hold only on the console's root-signed grant for |
| 337 | // its entry (spec §11, Astra impl-6 finding 5): every box can write both |
| 338 | // logs, so an unsigned or foreign entry is dropped here, with a line, |
| 339 | // before anything is cleared. The grant's epoch is checked against the |
| 340 | // book where the clear is (rollout.Manager.RetryHost, or |
| 341 | // failureHolds.retryHost on a box that declined the flag but honours its |
| 342 | // book). |
| 343 | func applyRetries(routes []registry.Route, work []scopedRestart, mem *restartMemory) { |
| 344 | for _, rq := range retryRequests(routes, work) { |
| 345 | if relayRootRetry(mem.pub, rq) { |
| 346 | continue |
| 347 | } |
| 348 | if mem.retryHost == nil { |
| 349 | log.Printf("agent: retry requested for %s, dropped: rollout manager not armed on this box, so nothing is held to clear", rq.host) |
| 350 | continue |
| 351 | } |
| 352 | g, why := verifyRetry(mem.pub, rq, time.Now()) |
| 353 | if why != "" { |
| 354 | log.Printf("agent: retry requested for %s, dropped: %s", rq.host, why) |
| 355 | continue |
| 356 | } |
| 357 | if mem.retryHost(rq.host, g.Build, g.Book, g.Epoch) { |
| 358 | next := "its next attempt runs as a canary" |
| 359 | if armedRollout() == nil { |
| 360 | next = "it starts the build as today" // a declined box runs no canary |
| 361 | } |
| 362 | log.Printf("agent: retry requested for %s: its failure hold on %s is cleared and %s", rq.host, g.Build, next) |
| 363 | } else { |
| 364 | log.Printf("agent: retry for %s: nothing cleared for %s at epoch %d of book %q (the restriction has changed since the grant, it was another book's grant, or this box has not saved its failure book yet)", rq.host, g.Build, g.Epoch, g.Book) |
| 365 | } |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | // verifyRetry is the box's check of one retry entry (spec §11, Paul's |
| 370 | // ruling on plan 6 Q4): the grant must verify against the root key and |
| 371 | // name this entry's account, app and host, within its window |
| 372 | // (release.VerifyHostRetry). The reason it gives is an operator's log line. |
| 373 | func verifyRetry(pub ed25519.PublicKey, rq retryRequest, now time.Time) (release.HostRetryGrant, string) { |
| 374 | switch { |
| 375 | case len(pub) == 0: |
| 376 | return release.HostRetryGrant{}, "this box has no root public key to verify the grant with" |
| 377 | case rq.signed == "": |
| 378 | return release.HostRetryGrant{}, "unsigned (only a console's signed grant clears a hold)" |
| 379 | } |
| 380 | g, err := release.VerifyHostRetry(pub, rq.signed, rq.acct, rq.app, rq.host, now) |
| 381 | if err != nil { |
| 382 | return release.HostRetryGrant{}, fmt.Sprintf("the grant did not verify for this entry: %v", err) |
| 383 | } |
| 384 | return g, "" |
| 385 | } |
| 386 | |
| 387 | // withoutRetries drops retry requests, and apps left with none, so a serial |
| 388 | // move that carried only retries asks root for no restart pass. |
| 389 | func withoutRetries(work []scopedRestart) []scopedRestart { |
| 390 | var out []scopedRestart |
| 391 | for _, w := range work { |
| 392 | var keep []release.RestartRequest |
| 393 | for _, req := range w.requests { |
| 394 | if req.Scope.Kind != release.RestartScopeRetry { |
| 395 | keep = append(keep, req) |
| 396 | } |
| 397 | } |
| 398 | if len(keep) > 0 { |
| 399 | out = append(out, scopedRestart{app: w.app, requests: keep}) |
| 400 | } |
| 401 | } |
| 402 | return out |
| 403 | } |
| 404 | |
| 405 | // execBounceable is the complement of what rootRestartRoute's systemctl arms |
| 406 | // can reach — no named unit, no unit backing — so the agent's half and the |
| 407 | // root half never cycle the same process twice. Split out of bounceExecRoutes |
| 408 | // so the narrowed half cannot drift from the app-wide one on which processes |
| 409 | // belong to whom. |
| 410 | func execBounceable(rt registry.Route) bool { |
| 411 | return rt.Unit == "" && rt.Backing != registry.BackingUnit |
| 412 | } |
| 413 | |
| 414 | // bounceScopedRoutes cycles the agent's OWN children for the routes the |
| 415 | // outstanding narrowed requests name. bounceExecRoutes' argument, one selector |
| 416 | // narrower — including on hibernation: Bounce cannot start anything, so a |
| 417 | // sleeping instance is untouched and a warm one gets the cycle that was asked |
| 418 | // for. |
| 419 | func bounceScopedRoutes(routes []registry.Route, work []scopedRestart, bounce func(host string) bool) int { |
| 420 | if bounce == nil || len(work) == 0 { |
| 421 | return 0 |
| 422 | } |
| 423 | n := 0 |
| 424 | for _, w := range work { |
| 425 | for _, host := range scopedHosts(routes, w.app, w.requests, execBounceable) { |
| 426 | if bounce(host) { |
| 427 | n++ |
| 428 | log.Printf("agent: narrowed restart requested: bounced exec instance %s", host) |
| 429 | } |
| 430 | } |
| 431 | } |
| 432 | return n |
| 433 | } |
| 434 | |
| 435 | // restartScopeLeg is the ROOT pass's narrowed half, called from inside |
| 436 | // restartPass under the SAME lock and against the same marks file: for every |
| 437 | // app whose request log has moved past this box's mark, cycle the routes those |
| 438 | // requests name. |
| 439 | // |
| 440 | // already is the hosts the app-wide half has cycled on this pass. A route that |
| 441 | // has just been restarted by a wider request must not be restarted a second |
| 442 | // time by a narrower one that arrived in the same window — one pass, one cycle |
| 443 | // per host. |
| 444 | // |
| 445 | // The rules are restartPass's, applied to this object: |
| 446 | // |
| 447 | // - An app this box has not recorded is recorded and NOT cycled. |
| 448 | // - An unknown baseline records and dispatches nothing. |
| 449 | // - A mark that rolled off the log re-baselines and dispatches nothing. |
| 450 | // - HIBERNATING routes are skipped and not counted as failures: restarting a |
| 451 | // process that is not running is meaningless, waking it to restart it |
| 452 | // defeats hibernation for no gain, and it execs fresh on its next wake. |
| 453 | // - A dispatch FAILURE still advances the mark. A request is a point-in-time |
| 454 | // event, not a desired state; retrying it every hour forever is how one |
| 455 | // wrong unit name becomes a permanent restart loop. |
| 456 | // - The mark advances only AFTER the dispatch, so a pass that dies mid-cycle |
| 457 | // retries rather than swallowing the request. |
| 458 | func restartScopeLeg(ctx context.Context, routes []registry.Route, store release.Store, apps [][2]string, |
| 459 | marks *restartMarks, known bool, restart func(host string) error, already map[string]bool) (changed, readFailed bool) { |
| 460 | for _, k := range apps { |
| 461 | g, _, err := release.LoadRestartScopes(ctx, store, k[0], k[1]) |
| 462 | if err != nil { |
| 463 | log.Printf("adopt: narrowed restarts %s/%s: %v (skipped this pass)", k[0], k[1], err) |
| 464 | readFailed = true |
| 465 | continue |
| 466 | } |
| 467 | newest := g.Newest() |
| 468 | key := restartMarkKey(k[0], k[1]) |
| 469 | prev, seen := marks.Scopes[key] |
| 470 | if !known || !seen { |
| 471 | if !seen || prev != newest { |
| 472 | marks.Scopes[key] = newest |
| 473 | changed = true |
| 474 | } |
| 475 | continue |
| 476 | } |
| 477 | if prev == newest { |
| 478 | continue |
| 479 | } |
| 480 | pending, found := g.PendingSince(prev) |
| 481 | if !found { |
| 482 | log.Printf("adopt: narrowed restarts for %s/%s: this box's position is no longer in the log — re-baselined, nothing cycled", k[0], k[1]) |
| 483 | marks.Scopes[key] = newest |
| 484 | changed = true |
| 485 | continue |
| 486 | } |
| 487 | dispatched := 0 |
| 488 | for _, host := range scopedHosts(routes, k, pending, func(registry.Route) bool { return true }) { |
| 489 | if already[host] { |
| 490 | continue |
| 491 | } |
| 492 | rt, ok := routeByHost(routes, host) |
| 493 | if !ok { |
| 494 | continue |
| 495 | } |
| 496 | if rt.Hibernate { |
| 497 | log.Printf("adopt: narrowed restart for %s/%s: %s is hibernating — left asleep (it execs fresh on its next wake)", k[0], k[1], host) |
| 498 | continue |
| 499 | } |
| 500 | if err := restart(host); err != nil { |
| 501 | log.Printf("adopt: narrowed restart for %s/%s: %s did not cycle: %v", k[0], k[1], host, err) |
| 502 | continue |
| 503 | } |
| 504 | already[host] = true |
| 505 | dispatched++ |
| 506 | } |
| 507 | // One line per APP, saying DISPATCHED rather than cycled, for the |
| 508 | // reason the app-wide half says it: an exec-backed route reaching |
| 509 | // rootRestartRoute lands on a try-restart of a carlos-app@ unit that |
| 510 | // does not exist — a deliberate no-op this pass cannot tell apart from |
| 511 | // a real one. |
| 512 | log.Printf("adopt: narrowed restart for %s/%s: %d request(s), dispatched to %d route(s)", k[0], k[1], len(pending), dispatched) |
| 513 | marks.Scopes[key] = newest |
| 514 | changed = true |
| 515 | } |
| 516 | return changed, readFailed |
| 517 | } |
| 518 | |
| 519 | // routeByHost finds a route by host among the ones this box serves. Linear |
| 520 | // because the route set is a box's own table, and a map built per call would |
| 521 | // be the same walk with an allocation on top. |
| 522 | func routeByHost(routes []registry.Route, host string) (registry.Route, bool) { |
| 523 | for _, rt := range routes { |
| 524 | if rt.Host == host { |
| 525 | return rt, true |
| 526 | } |
| 527 | } |
| 528 | return registry.Route{}, false |
| 529 | } |
| 530 | |
amadan