| 1 | package host |
| 2 | |
| 3 | // steering_loop.go — the DNS steering converger's housekeeper loop |
| 4 | // (multi-region ingress spec, Part 4; plan 2026-08-23-multi-region-steering |
| 5 | // Task 6). The converger itself lives in internal/steering; this file is the |
| 6 | // wiring: what it reads, when it runs, and what it refuses to run on. |
| 7 | |
| 8 | import ( |
| 9 | "context" |
| 10 | "fmt" |
| 11 | "log" |
| 12 | "math/rand/v2" |
| 13 | "os" |
| 14 | "sort" |
| 15 | "strings" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/provision" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | "github.com/carlosframework/platform/internal/steering" |
| 22 | ) |
| 23 | |
| 24 | // loadEdgesFileBodies reads every box's edges file under control/edges/ and |
| 25 | // returns the raw bodies in listing order (sorted by key, so the fold |
| 26 | // downstream is stable tick to tick). |
| 27 | // |
| 28 | // FAIL-CLOSED, and this is the whole point of the function: a listing that |
| 29 | // errors, or ANY object that lists but will not Get, returns an ERROR and no |
| 30 | // bodies. The alternative — carrying on with what came back — is the SHORT |
| 31 | // READ that Task 5's shrink guard was added to survive: the fold would be |
| 32 | // missing a live box's edge, every record belonging to that box would look |
| 33 | // managed-but-unwanted, and the pass would delete them. NXDOMAIN for a live |
| 34 | // steered host, caused by a bucket blip rather than by anything an operator |
| 35 | // did. |
| 36 | // |
| 37 | // This is deliberately STRICTER than internal/edge's homeMap, which reads the |
| 38 | // same prefix and tolerates a single unreadable object by carrying that box's |
| 39 | // previous parse forward. homeMap can do that because it HOLDS a last-good |
| 40 | // snapshot; this pass holds nothing between ticks, so "tolerate" here would |
| 41 | // mean "silently converge on less". Even a raced delete (list saw the key, |
| 42 | // Get says it is gone — a box retired mid-pass) fails the pass: the cost is |
| 43 | // one skipped tick, five minutes later the listing no longer names it, and |
| 44 | // the withdrawal happens on complete information instead of a race. |
| 45 | // |
| 46 | // A key that is not <box>.json directly under the prefix is not read at all — |
| 47 | // the same key-shape filter homeMap applies. A stray object under the prefix |
| 48 | // is not this deployment's edges file, and must not be able to fail the pass |
| 49 | // forever. |
| 50 | func loadEdgesFileBodies(ctx context.Context, s release.Store) ([][]byte, error) { |
| 51 | keys, err := s.List(ctx, edgesFilePrefix) |
| 52 | if err != nil { |
| 53 | return nil, fmt.Errorf("steering: list %s: %w", edgesFilePrefix, err) |
| 54 | } |
| 55 | sort.Strings(keys) |
| 56 | out := make([][]byte, 0, len(keys)) |
| 57 | for _, key := range keys { |
| 58 | name, ok := strings.CutPrefix(key, edgesFilePrefix) |
| 59 | if !ok || !strings.HasSuffix(name, ".json") || strings.Contains(name, "/") { |
| 60 | continue |
| 61 | } |
| 62 | body, _, err := s.Get(ctx, key) |
| 63 | if err != nil { |
| 64 | return nil, fmt.Errorf("steering: read %s: %w", key, err) |
| 65 | } |
| 66 | out = append(out, body) |
| 67 | } |
| 68 | return out, nil |
| 69 | } |
| 70 | |
| 71 | // steeredRoutesFromRecords reads the deployment's steered hosts from the |
| 72 | // BUCKET RECORDS — provision.ListAllRecords — and never from this box's |
| 73 | // registry. |
| 74 | // |
| 75 | // The registry is the wrong source and the bug is not theoretical (review |
| 76 | // finding, 2026-08-23). Routes are NOT uniformly provisioned: an |
| 77 | // everywhereClass record (exec/unit/socket kinds) is homed to ONE box, so |
| 78 | // that host is absent from every other box's registry. This duty runs on |
| 79 | // whichever box holds the housekeeper lease — so a steered, home-boxed host |
| 80 | // read from the local registry would be present in the desired set while the |
| 81 | // lease sat on its home box and ABSENT the moment the lease moved. Absent |
| 82 | // means managed-but-unwanted, which means DELETED: a live host goes NXDOMAIN |
| 83 | // until the lease comes back, and the shrink guard does not fire at 2-of-8. |
| 84 | // DNS bound to housekeeper elections is not a system anyone can operate. |
| 85 | // The records are the deployment's truth, identical from every box. |
| 86 | // |
| 87 | // The mapping is deliberately MINIMAL: desiredState (internal/steering) |
| 88 | // reads exactly three fields off a registry.Route — Steering (the "latency" |
| 89 | // opt-in), Host (the DNS name, and the wildcard check) and Pool (steering |
| 90 | // requires the shared pool) — verified field by field against that fold, so |
| 91 | // carrying more would be inventing agreement the fold does not need. |
| 92 | // |
| 93 | // Records with Steering == "" are dropped HERE rather than passed down: the |
| 94 | // fold skips them SILENTLY anyway (steeredHosts' `case "": continue`), so |
| 95 | // nothing an operator would see is lost. Every other value — including |
| 96 | // invalid ones, wildcard hosts and pool-scoped rows — is passed through |
| 97 | // untouched so the fold's own loud, documented refusals still fire. This |
| 98 | // loader must not become a second, quieter policy gate. |
| 99 | func steeredRoutesFromRecords(ctx context.Context, s release.Store) ([]registry.Route, error) { |
| 100 | recs, err := provision.ListAllRecords(ctx, s) |
| 101 | if err != nil { |
| 102 | return nil, err |
| 103 | } |
| 104 | out := make([]registry.Route, 0, len(recs)) |
| 105 | for _, rec := range recs { |
| 106 | if rec.Steering == "" { |
| 107 | continue |
| 108 | } |
| 109 | out = append(out, registry.Route{Host: rec.Host, Steering: rec.Steering, Pool: rec.Pool}) |
| 110 | } |
| 111 | return out, nil |
| 112 | } |
| 113 | |
| 114 | // steeringPass is one converge attempt: read both inputs from the STORE, |
| 115 | // then converge. The registry is not consulted at all — see |
| 116 | // steeredRoutesFromRecords. |
| 117 | // |
| 118 | // Both reads fail closed (see loadEdgesFileBodies and ListAllRecords) and |
| 119 | // BOTH must succeed before converge is called even once — a records read |
| 120 | // that failed is not "no steered hosts" any more than a failed edges listing |
| 121 | // is "no edges". converge is injected rather than called directly so this |
| 122 | // contract is testable without an AWS client: the real one is the closure |
| 123 | // steeringLoop builds around steering.ConvergeDeployment. |
| 124 | func steeringPass(ctx context.Context, s release.Store, converge func(context.Context, []registry.Route, [][]byte) error) error { |
| 125 | edges, err := loadEdgesFileBodies(ctx, s) |
| 126 | if err != nil { |
| 127 | return err |
| 128 | } |
| 129 | // The records through the record view when the change index is armed |
| 130 | // and the view can vouch; the bucket otherwise (viewStore). |
| 131 | rs, _, _ := viewStore(ctx, s) |
| 132 | routes, err := steeredRoutesFromRecords(ctx, rs) |
| 133 | if err != nil { |
| 134 | return err |
| 135 | } |
| 136 | return converge(ctx, routes, edges) |
| 137 | } |
| 138 | |
| 139 | // steeringDuty wraps one pass as a housekeeper duty. |
| 140 | // |
| 141 | // DORMANCY (zoneID == "") is enforced here, at the top, and it is a |
| 142 | // structural claim rather than a promise: pass is the closure holding the |
| 143 | // store handle and the AWS client — every read and every API call this duty |
| 144 | // can make — so a dormant duty that never calls it does neither, by |
| 145 | // construction. Every box in production today is in exactly this state. |
| 146 | // |
| 147 | // DUTY-STATUS HONESTY — read this before trusting a green tick. A nil return |
| 148 | // from this duty does NOT mean the zone converged. internal/steering's two |
| 149 | // safety refusals (the empty-desired-set refusal and the proportional shrink |
| 150 | // guard) both log loudly and return nil, deliberately: the duty cannot fix |
| 151 | // either, and marking it failed every five minutes would bury the one line |
| 152 | // that says what actually happened. So a refused pass is indistinguishable |
| 153 | // from a clean one at THIS layer, by design, and the operator surface for |
| 154 | // steering is the JOURNAL — `journalctl -u carlos -g "^steering:"` — not the |
| 155 | // housekeeper's error log. |
| 156 | // |
| 157 | // The case with no escape hatch, recorded for the rollout notes: a GENUINE |
| 158 | // withdrawal of more than half the managed records (a real multi-box |
| 159 | // retirement) is refused on every tick, forever, with no flag to override it. |
| 160 | // Recovery is an operator deleting the records by hand, or withdrawing the |
| 161 | // boxes in batches under the limit. Threading a status back up through the |
| 162 | // duty was considered and left out of this task on purpose — it changes the |
| 163 | // housekeeperDuty contract for every duty, and the journal already carries |
| 164 | // the truth. |
| 165 | // The pass == nil half of the guard is one token of totality on a path that |
| 166 | // writes production DNS: steeringLoop leaves pass nil in exactly the branch |
| 167 | // that leaves zoneID empty, so the two conditions are the same condition |
| 168 | // today — but "today" is doing load-bearing work in that sentence, and the |
| 169 | // failure mode if it ever stopped being true is a nil call that panics the |
| 170 | // agent's goroutine rather than a duty that declines to run. |
| 171 | func steeringDuty(zoneID string, pass func(context.Context) error) housekeeperDuty { |
| 172 | return housekeeperDuty{"steering converge", func(ctx context.Context) error { |
| 173 | if zoneID == "" || pass == nil { |
| 174 | return nil |
| 175 | } |
| 176 | return pass(ctx) |
| 177 | }} |
| 178 | } |
| 179 | |
| 180 | // steeringLoop converges the deployment's steering zone every 5 minutes + |
| 181 | // jitter, on the one box holding the housekeeper lease. |
| 182 | // |
| 183 | // Its shape is mintMeshKeyLoop's, deliberately and for the same reason |
| 184 | // (housekeeper.go's own comment): its OWN loop, gated on nothing but |
| 185 | // deployment-store presence, never a duty folded into retentionLoop's tick — |
| 186 | // that loop never arms at all on a box missing CARLOS_RELEASE_PUBKEY, and |
| 187 | // such a box still WINS the housekeeper election, so a steering duty living |
| 188 | // there would silently never converge, fleet-wide, with no error anywhere. |
| 189 | // |
| 190 | // Env, read ONCE at arm (domainsLoop's read-once precedent): |
| 191 | // |
| 192 | // - CARLOS_STEERING_ZONE_ID — the zone. Unset, blank, or malformed and the |
| 193 | // loop still ARMS but its duty is dormant: one log line here, and nothing |
| 194 | // else, ever. Malformed is logged loudly by SteeringZoneID itself and is |
| 195 | // dormancy, never a crash loop — a typo must not take a box's agent down. |
| 196 | // - PLATFORM_APPS_DOMAIN — the domain the synthetic edge.<apps-domain> |
| 197 | // steering hostname is built from. Empty is not fatal: the fold logs it |
| 198 | // and mints no steering hostname (steered per-host records are |
| 199 | // unaffected). |
| 200 | // |
| 201 | // FIRST REAL BEHAVIOUR, when a deployment does set the zone id while still |
| 202 | // running ONE box: the steering hostname and every steered host get a |
| 203 | // single-edge latency record — the same answers today's static DNS gives — |
| 204 | // plus one Route53 health check per EIP, which is what adds rung-2 |
| 205 | // withdrawal for steered hosts even single-box. Nothing about it needs a |
| 206 | // second region to be correct. |
| 207 | func steeringLoop() { |
| 208 | ctx := context.Background() |
| 209 | if shouldSkipPublisherElection(os.Getenv) { |
| 210 | log.Printf("agent: steering loop is a platform-box job — skipped on a fleet box") |
| 211 | return |
| 212 | } |
| 213 | store, err := openDeployStore(ctx) |
| 214 | if err != nil { |
| 215 | log.Printf("agent: steering loop disabled: %v", err) |
| 216 | return |
| 217 | } |
| 218 | |
| 219 | zoneID := steering.SteeringZoneID() |
| 220 | var pass func(context.Context) error |
| 221 | if zoneID == "" { |
| 222 | log.Printf("agent: steering: zone unset — dormant (CARLOS_STEERING_ZONE_ID is unset or unusable; the duty reads nothing and calls no AWS API)") |
| 223 | log.Printf("agent: steering loop armed (5min + jitter) — dormant") |
| 224 | } else { |
| 225 | // Constructed only on a deployment that actually steers: the default |
| 226 | // credential chain is the box instance profile, and a dormant box has |
| 227 | // no business resolving credentials at all. |
| 228 | client, err := steering.NewClient(ctx) |
| 229 | if err != nil { |
| 230 | log.Printf("agent: steering loop disabled: %v", err) |
| 231 | return |
| 232 | } |
| 233 | appsDomain := strings.TrimSpace(os.Getenv("PLATFORM_APPS_DOMAIN")) |
| 234 | pass = func(ctx context.Context) error { |
| 235 | return steeringPass(ctx, store, func(ctx context.Context, routes []registry.Route, edges [][]byte) error { |
| 236 | return steering.ConvergeDeployment(ctx, client, zoneID, routes, edges, appsDomain) |
| 237 | }) |
| 238 | } |
| 239 | log.Printf("agent: steering loop armed (5min + jitter), zone %s, apps domain %q — converges only while this box holds the housekeeper lease", zoneID, appsDomain) |
| 240 | } |
| 241 | |
| 242 | duty := steeringDuty(zoneID, pass) |
| 243 | delay := time.Minute |
| 244 | for { |
| 245 | time.Sleep(delay) |
| 246 | delay = 5*time.Minute + rand.N(time.Minute) |
| 247 | runHousekeeperDuties(housekeeperGate, duty) |
| 248 | } |
| 249 | } |
| 250 | |
amadan