| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "log" |
| 6 | "slices" |
| 7 | "sync/atomic" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/hostclaim" |
| 10 | "github.com/carlosframework/platform/internal/provision" |
| 11 | "github.com/carlosframework/platform/internal/registry" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | "github.com/carlosframework/platform/internal/schedule" |
| 14 | ) |
| 15 | |
| 16 | // movesHoldView is what this box's other passes ask about running moves |
| 17 | // (moves spec §5.2, §9.4, §9.6, §19 K9): reconcile mints and keeps a |
| 18 | // participant's row through hold and keep, and the schedule converger |
| 19 | // leaves a frozen host's timers alone through frozen. Strictness is not |
| 20 | // here: the moves worker's setStrict is the one strict source (G9). |
| 21 | // |
| 22 | // It is rebuilt from every moves pass's record list, plus one claim read |
| 23 | // per host with a Phase 1 move in flight, and read from the reconcile and |
| 24 | // schedule loops in between. It fails closed: until the first build, keep |
| 25 | // and frozen answer true for every host (a removal or a schedule converge |
| 26 | // waits a pass rather than act on a host it cannot vouch for), and a host |
| 27 | // whose claim would not read is kept and frozen until it does. hold judges |
| 28 | // the record the caller passes, not the view's copy, so a flip the view has |
| 29 | // not seen yet still mints the landing row held. |
| 30 | // |
| 31 | // Only built with the Phase 1 switch on: off, armedHoldView is nil and no |
| 32 | // pass is given a hook. |
| 33 | // |
| 34 | // Ending a hold that hold would re-assert (the landing's open, the |
| 35 | // finalisation: Task 10, movesland.go) must happen under |
| 36 | // provision.WithHostHoldLock, the lock reconcile's hold decision holds from |
| 37 | // its record re-read through its conditional write. See that function. |
| 38 | type movesHoldView struct { |
| 39 | box, region string |
| 40 | read func(ctx context.Context, s release.Store, host string) (hostclaim.Observed, error) |
| 41 | // written is reconcile's HoldWritten: the stage-requested marker after |
| 42 | // a hold write (G10). Nil writes nothing. |
| 43 | written func(host string) |
| 44 | // gate is the schedule wiring's HostGate, set by armScheduleHolds; nil |
| 45 | // until then (no scheduled work armed: nothing to drain). |
| 46 | gate atomic.Pointer[schedule.HostGate] |
| 47 | snap atomic.Pointer[holdSnapshot] |
| 48 | } |
| 49 | |
| 50 | // holdSnapshot is one build: what each in-flight move's claim said. |
| 51 | type holdSnapshot struct { |
| 52 | claims map[string]holdClaim // host -> the claim's op, for hosts whose claim read |
| 53 | unknown map[string]bool // hosts whose claim would not read |
| 54 | } |
| 55 | |
| 56 | type holdClaim struct { |
| 57 | kind string |
| 58 | participants []string |
| 59 | } |
| 60 | |
| 61 | func newMovesHoldView(box, region string, read func(context.Context, release.Store, string) (hostclaim.Observed, error), written func(host string)) *movesHoldView { |
| 62 | return &movesHoldView{box: box, region: region, read: read, written: written} |
| 63 | } |
| 64 | |
| 65 | // moveKinds are the claim op kinds a move holds (§19.1). Region HA's |
| 66 | // relocating is its own op and freezes through its own wiring. |
| 67 | var moveKinds = map[string]bool{hostclaim.KindMoving: true, hostclaim.KindLanding: true, |
| 68 | hostclaim.KindReverting: true, hostclaim.KindRestoring: true} |
| 69 | |
| 70 | // refresh rebuilds the view from recs: every host with a Phase 1 move in |
| 71 | // flight has its claim read. Called only with a list that was read; a |
| 72 | // failed listing leaves the last build in place. |
| 73 | func (v *movesHoldView) refresh(ctx context.Context, s release.Store, recs []provision.Record) { |
| 74 | next := &holdSnapshot{claims: map[string]holdClaim{}, unknown: map[string]bool{}} |
| 75 | for _, rec := range recs { |
| 76 | if rec.Move == nil || rec.Move.Legacy() { |
| 77 | continue |
| 78 | } |
| 79 | if _, seen := next.claims[rec.Host]; seen || next.unknown[rec.Host] { |
| 80 | continue |
| 81 | } |
| 82 | obs, err := v.read(ctx, s, rec.Host) |
| 83 | if err != nil { |
| 84 | log.Printf("moves: %s: claim unreadable (%v); its row is kept and its timers frozen until it reads", rec.Host, err) |
| 85 | next.unknown[rec.Host] = true |
| 86 | continue |
| 87 | } |
| 88 | if obs.Absent || obs.Claim == nil { |
| 89 | continue |
| 90 | } |
| 91 | next.claims[rec.Host] = holdClaim{kind: obs.Claim.Op.Kind, participants: slices.Clone(obs.Claim.Op.Participants)} |
| 92 | } |
| 93 | v.snap.Store(next) |
| 94 | } |
| 95 | |
| 96 | // keep is reconcile's KeepParticipant: this box is a participant of a |
| 97 | // move-kind op on host. |
| 98 | func (v *movesHoldView) keep(host string) bool { |
| 99 | snap := v.snap.Load() |
| 100 | if snap == nil || snap.unknown[host] { |
| 101 | return true |
| 102 | } |
| 103 | c, ok := snap.claims[host] |
| 104 | return ok && moveKinds[c.kind] && slices.Contains(c.participants, v.box) |
| 105 | } |
| 106 | |
| 107 | // frozen is the converger's Frozen: a move-kind op holds host's claim. |
| 108 | func (v *movesHoldView) frozen(host string) bool { |
| 109 | snap := v.snap.Load() |
| 110 | if snap == nil || snap.unknown[host] { |
| 111 | return true |
| 112 | } |
| 113 | c, ok := snap.claims[host] |
| 114 | return ok && moveKinds[c.kind] |
| 115 | } |
| 116 | |
| 117 | // The roles a box plays in one move. |
| 118 | const ( |
| 119 | roleNone = "" |
| 120 | roleSource = "source" |
| 121 | roleTarget = "target" |
| 122 | ) |
| 123 | |
| 124 | // role is this box's part in rec's move: by the claim's participants |
| 125 | // ({source, target}, §19.1) when the view read the claim, else by the |
| 126 | // record's regions. A box the claim does not name plays none. |
| 127 | func (v *movesHoldView) role(rec provision.Record) (role, claimKind string) { |
| 128 | if snap := v.snap.Load(); snap != nil { |
| 129 | if c, ok := snap.claims[rec.Host]; ok && moveKinds[c.kind] && len(c.participants) == 2 { |
| 130 | switch v.box { |
| 131 | case c.participants[0]: |
| 132 | return roleSource, c.kind |
| 133 | case c.participants[1]: |
| 134 | return roleTarget, c.kind |
| 135 | } |
| 136 | return roleNone, c.kind |
| 137 | } |
| 138 | } |
| 139 | m := rec.Move |
| 140 | switch { |
| 141 | case m.From == v.region && m.To != v.region: |
| 142 | return roleSource, "" |
| 143 | case m.To == v.region && m.From != v.region: |
| 144 | return roleTarget, "" |
| 145 | case m.To == v.region: |
| 146 | // Both ends in this region and no claim to say which box is which: |
| 147 | // the state decides, and either answer holds the row. |
| 148 | switch m.State { |
| 149 | case provision.MoveLanding, provision.MoveCommitted: |
| 150 | return roleTarget, "" |
| 151 | } |
| 152 | return roleSource, "" |
| 153 | } |
| 154 | return roleNone, "" |
| 155 | } |
| 156 | |
| 157 | // hold is reconcile's Hold: the status and op a participant's row is held |
| 158 | // under, from the record's state and this box's role (§5.2, §7.4, §7.6, |
| 159 | // §7.8, §19.1): the source moving from stopping until its row is deleted at |
| 160 | // finalization, restoring through a revert or a rollback; the target landing |
| 161 | // from the flip until the commit opens its row, releasing through a |
| 162 | // rollback. A rollback's holds are under the rollback id; a rollback that |
| 163 | // has none (malformed) holds under the move id rather than not at all. |
| 164 | func (v *movesHoldView) hold(rec provision.Record) (status, op string, ok bool) { |
| 165 | m := rec.Move |
| 166 | if m == nil || m.Legacy() { |
| 167 | return "", "", false |
| 168 | } |
| 169 | role, kind := v.role(rec) |
| 170 | rollbackOp := m.ID |
| 171 | if m.Rollback != nil && m.Rollback.ID != "" { |
| 172 | rollbackOp = m.Rollback.ID |
| 173 | } |
| 174 | switch role { |
| 175 | case roleSource: |
| 176 | switch m.State { |
| 177 | case provision.MoveCutting: |
| 178 | switch m.CutPhase { |
| 179 | case provision.CutStopping, provision.CutStopped, provision.CutStaged, provision.CutArchived, provision.CutFlipped: |
| 180 | return "moving", m.ID, true |
| 181 | } |
| 182 | case provision.MoveLanding, provision.MoveCommitted: |
| 183 | return "moving", m.ID, true |
| 184 | case provision.MoveReverting: |
| 185 | return "restoring", m.ID, true |
| 186 | case provision.MoveAborting: |
| 187 | // Aborting follows a revert (claim reverting, the row held |
| 188 | // restoring) or comes before any cut (claim moving, the row |
| 189 | // serving): only the first is held, and only the claim can say. |
| 190 | if kind == hostclaim.KindReverting { |
| 191 | return "restoring", m.ID, true |
| 192 | } |
| 193 | case provision.MoveRollingBack, provision.MoveStuck: |
| 194 | return "restoring", rollbackOp, true |
| 195 | } |
| 196 | case roleTarget: |
| 197 | switch m.State { |
| 198 | case provision.MoveLanding: |
| 199 | return "landing", m.ID, true |
| 200 | case provision.MoveRollingBack, provision.MoveStuck: |
| 201 | return "releasing", rollbackOp, true |
| 202 | } |
| 203 | } |
| 204 | return "", "", false |
| 205 | } |
| 206 | |
| 207 | // drainTimers is the cut's timer drain (moves spec §9.4 "drained", Task 5's |
| 208 | // moveCfg.TimersDrained): after SetHold, it waits until no delivery or |
| 209 | // convergence that could have missed the hold is still running for host. |
| 210 | // No gate (no scheduled work armed on this box) has nothing to drain. |
| 211 | func (v *movesHoldView) drainTimers(ctx context.Context, host string) error { |
| 212 | return v.gate.Load().Drain(ctx, host) |
| 213 | } |
| 214 | |
| 215 | // armedHoldView is the armed worker's hold view, or nil when no worker is |
| 216 | // armed or the Phase 1 switch is off. |
| 217 | func armedHoldView() *movesHoldView { |
| 218 | w := armedMovesWorker.Load() |
| 219 | if w == nil || !w.cfg.Enabled { |
| 220 | return nil |
| 221 | } |
| 222 | return w.holds |
| 223 | } |
| 224 | |
| 225 | // withMovesHolds gives a reconcile pass the hold view's hooks, unless the |
| 226 | // caller set its own. With the switch off it returns cfg unchanged: main. |
| 227 | func withMovesHolds(cfg provision.ReconcileConfig) provision.ReconcileConfig { |
| 228 | v := armedHoldView() |
| 229 | if v == nil || cfg.Hold != nil || cfg.KeepParticipant != nil { |
| 230 | return cfg |
| 231 | } |
| 232 | cfg.Hold, cfg.KeepParticipant = v.hold, v.keep |
| 233 | if v.written != nil && cfg.HoldWritten == nil { |
| 234 | cfg.HoldWritten = v.written |
| 235 | } |
| 236 | return cfg |
| 237 | } |
| 238 | |
| 239 | // armScheduleHolds wires the schedule halves to the hold view: one HostGate |
| 240 | // shared by the runner, the converger (across both of the agent's converge |
| 241 | // loops: there is one Converger) and the cut's drain, and the converger's |
| 242 | // freeze. v nil (switch off) or sd nil (no scheduled work) leaves both as |
| 243 | // main has them. |
| 244 | func armScheduleHolds(sd *scheduleDeps, v *movesHoldView) { |
| 245 | if sd == nil || v == nil { |
| 246 | return |
| 247 | } |
| 248 | gate := &schedule.HostGate{} |
| 249 | sd.runner.Gate = gate |
| 250 | if sd.conv != nil { |
| 251 | sd.conv.Gate = gate |
| 252 | sd.conv.Frozen = v.frozen |
| 253 | } |
| 254 | v.gate.Store(gate) |
| 255 | } |
| 256 | |
| 257 | // refreshMovesHolds builds the hold view once, outside a moves pass: the |
| 258 | // agent's startup converge runs before the first moves pass, and an unbuilt |
| 259 | // view freezes every host. A failed listing leaves it unbuilt; the first |
| 260 | // moves pass that lists builds it. |
| 261 | func refreshMovesHolds(ctx context.Context, s release.Store) { |
| 262 | v := armedHoldView() |
| 263 | if v == nil || s == nil { |
| 264 | return |
| 265 | } |
| 266 | recs, err := provision.ListAllRecords(ctx, s) |
| 267 | if err != nil { |
| 268 | log.Printf("moves: hold view: list records: %v; rows are kept and timers frozen until a moves pass lists them", err) |
| 269 | return |
| 270 | } |
| 271 | v.refresh(ctx, s, recs) |
| 272 | } |
| 273 | |
| 274 | // holdStageRequester is the hold view's HoldWritten: the stage-requested |
| 275 | // marker, under its own throttle, so root re-judges what it staged under |
| 276 | // the new hold (registry.SetHold's contract, G10). |
| 277 | func holdStageRequester(dd string) func(host string) { |
| 278 | req := markerRequester(dd, func(host, _ string, err error) { |
| 279 | if err != nil { |
| 280 | log.Printf("agent: %s: held for a move, but cannot request a root stage pass: %v", host, err) |
| 281 | return |
| 282 | } |
| 283 | log.Printf("agent: %s: held for a move; asked root to re-judge what it staged", host) |
| 284 | }) |
| 285 | return func(host string) { req(host, "", nil) } |
| 286 | } |
| 287 | |
| 288 | // handHold is the box's hand-over of a move's hold to its rollback (ruling |
| 289 | // J3): registry.HandHold under WithHostHoldLock, then written (in production |
| 290 | // holdStageRequester) only when the hand-over landed, so root re-judges what |
| 291 | // it staged under the new hold. A refusal, an absent row or a replay touches |
| 292 | // nothing. Every caller hands a hold over through this wrapper, never through |
| 293 | // registry.HandHold directly, which cannot reach the marker. |
| 294 | // |
| 295 | // validate, when not nil, is the caller's own fresh check (Task 4 fix round |
| 296 | // 1, ruling 1; J3's record check): it runs under the hold lock, after the |
| 297 | // row is re-read and before anything is decided, and an error from it moves |
| 298 | // nothing and is returned. It runs inside WithHostHoldLock, so it must not |
| 299 | // take that lock again; a provision record read is what it is for. |
| 300 | // |
| 301 | // true means the row is held status under toOp: the hand-over landed, or a |
| 302 | // replay found it already done. false with no error means nothing matched: |
| 303 | // the row is held by some other op, unheld, or absent. |
| 304 | func handHold(reg *registry.Registry, written func(host string)) func(host, status, toOp, fromOp string, validate func() error) (bool, error) { |
| 305 | return func(host, status, toOp, fromOp string, validate func() error) (bool, error) { |
| 306 | var handed bool |
| 307 | var err error |
| 308 | provision.WithHostHoldLock(host, func() { |
| 309 | var rt *registry.Route |
| 310 | if rt, err = reg.Get(host); err != nil || rt == nil { |
| 311 | return |
| 312 | } |
| 313 | if validate != nil { |
| 314 | if err = validate(); err != nil { |
| 315 | return |
| 316 | } |
| 317 | } |
| 318 | if rt.HoldOp == toOp && rt.Status == status && toOp != "" { |
| 319 | handed = true |
| 320 | return |
| 321 | } |
| 322 | if handed, err = reg.HandHold(host, status, toOp, fromOp); err != nil || !handed { |
| 323 | return |
| 324 | } |
| 325 | if written != nil { |
| 326 | written(host) |
| 327 | } |
| 328 | }) |
| 329 | return handed, err |
| 330 | } |
| 331 | } |
| 332 | |
amadan