| 1 | package hostbox |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "slices" |
| 6 | "time" |
| 7 | |
| 8 | "github.com/carlosframework/platform/internal/hostclaim" |
| 9 | "github.com/carlosframework/platform/internal/hostguard" |
| 10 | "github.com/carlosframework/platform/internal/hostwrite" |
| 11 | "github.com/carlosframework/platform/internal/park" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | ) |
| 14 | |
| 15 | type RecordSelection struct { |
| 16 | Host, Key, ETag, Incarnation, Family string |
| 17 | Layout string |
| 18 | Paths []string |
| 19 | DryRun bool |
| 20 | } |
| 21 | type Reconciler struct { |
| 22 | PrepareInventory func(context.Context, RecordSelection, hostguard.Captured) error |
| 23 | Prepare func(context.Context, hostclaim.Observed) error |
| 24 | HomeRegion string |
| 25 | Claims release.Store |
| 26 | Guard hostguard.Guard |
| 27 | Local park.Local |
| 28 | Writes hostwrite.Config |
| 29 | Box, Region string |
| 30 | Now func() time.Time |
| 31 | // CheckRecord directly checks the selected record ETag and ownership acquisition. |
| 32 | CheckRecord func(context.Context, RecordSelection) error |
| 33 | // CheckLocal includes complete census and journal recovery before initialization. |
| 34 | CheckLocal func(context.Context, hostclaim.Observed, RecordSelection) error |
| 35 | // CheckHeld is the hold-only path's local check, in CheckLocal's place |
| 36 | // there: this box's own existing installation for the row it would |
| 37 | // hold, never the claim holder's stamp (Task 10b, Astra round 1). |
| 38 | // Production: EmptyInstallationCensus.CheckHeld, which |
| 39 | // ReconcileAdapter.Install sets from its census. nil refuses the hold. |
| 40 | CheckHeld func(context.Context, hostclaim.Observed, RecordSelection) error |
| 41 | // Adopt is Plan 5's #232 backfill coordinator, never a second adoption rule. |
| 42 | Adopt func(context.Context, RecordSelection) (hostclaim.Observed, error) |
| 43 | // FenceClosed reports whether this box's admission fence for the host |
| 44 | // is closed on the claim given (region HA §5.3: the source in |
| 45 | // releasing is ready until then). It is asked only where the row |
| 46 | // decision depends on it (rowDecision). It is called under the host lock |
| 47 | // and must not take it (hostguard.Guard.AdmissionFenced). nil: never |
| 48 | // closed, today's behaviour. An error keeps rows and mints none (C6). |
| 49 | // Production: AdmissionFenceClosed, region HA on only (Task 10b). |
| 50 | FenceClosed func(context.Context, hostclaim.Observed) (bool, error) |
| 51 | // HoldRow keeps an existing row held for an operation without |
| 52 | // installation authority (ruling 17). Production: RegistryHoldRow, |
| 53 | // region HA on only (Task 10b). |
| 54 | HoldRow func(ctx context.Context, host, holdOp string) error |
| 55 | } |
| 56 | |
| 57 | // relocationAware is region HA on: Task 10b sets FenceClosed and HoldRow |
| 58 | // only with CARLOS_REGION_HA (host's relocationSourceRows). With neither, |
| 59 | // reconcile is today's: the serving executor alone (Serving), and no |
| 60 | // hold-only path, the completed relocation's catch-up hold included. |
| 61 | func (r Reconciler) relocationAware() bool { return r.FenceClosed != nil || r.HoldRow != nil } |
| 62 | |
| 63 | // relocates is the row decision's scope: a relocating claim with region HA |
| 64 | // on. Every other operation, moving included, keeps main's path (Serving), |
| 65 | // HA on or off: moves owns its holds, in provision's pass. The one |
| 66 | // exception is a serving claim's completed relocation, whose catch-up hold |
| 67 | // (completedHold) is decided apart from this scope. |
| 68 | func (r Reconciler) relocates(c *hostclaim.Claim) bool { |
| 69 | return r.relocationAware() && c != nil && c.Op.Kind == hostclaim.KindRelocating |
| 70 | } |
| 71 | |
| 72 | // completedHold is the catch-up hold (L18 joint round, finding 3): the |
| 73 | // source's fence closed in releasing, but its next reconcile ran only once |
| 74 | // the target had completed, so the claim serves on another box and |
| 75 | // rowDecision's scope (relocates) no longer reaches it, and main's Serving |
| 76 | // would refuse with the row left ready, which the edge serves, until the |
| 77 | // park-local retires it. Completion waits on no acknowledgement of the |
| 78 | // source's hold (ruling 80), so reconcile catches up here: the row is held |
| 79 | // for the completed op, a RowRetiring hold, until the park-local retires |
| 80 | // it (Task 10b decision 3: nothing un-holds). |
| 81 | // |
| 82 | // In scope only with region HA on, and only on positive evidence on the |
| 83 | // fresh claim: a serving op on another box (not this one, which serves |
| 84 | // through Serving) that journals a completed relocation (not completed on |
| 85 | // this box) and a park-local this box owes for exactly that op, which |
| 86 | // CompleteRelocation queues for the source and every installed box but |
| 87 | // the target. A box on lost is left to its park-local (decision 3). A |
| 88 | // park-local for another op, or none, is out of scope and main's path |
| 89 | // runs. A completion record that cannot be read is its error, never "no |
| 90 | // completion": the row is kept and nothing is minted. The path is hold |
| 91 | // only (Install, checkOn): no activation, no inventory, no execution. |
| 92 | func (r Reconciler) completedHold(c *hostclaim.Claim) (RowDecision, bool, error) { |
| 93 | if !r.relocationAware() || c == nil || c.Holder == nil || r.Box == "" || c.Op.Kind != hostclaim.KindServing || c.Op.Exec == r.Box || slices.Contains(c.Lost, r.Box) { |
| 94 | return RowDecision{}, false, nil |
| 95 | } |
| 96 | done, ok, err := hostclaim.ReadCompletedRelocation(c) |
| 97 | if err != nil { |
| 98 | return RowDecision{}, false, err |
| 99 | } |
| 100 | if !ok || done.Stop.Box == r.Box { |
| 101 | return RowDecision{}, false, nil |
| 102 | } |
| 103 | for _, o := range c.Obligations { |
| 104 | if o.Kind == "park-local" && o.Box == r.Box && o.Op == done.Op { |
| 105 | return RowDecision{Case: RowRetiring, HoldOp: done.Op}, true, nil |
| 106 | } |
| 107 | } |
| 108 | return RowDecision{}, false, nil |
| 109 | } |
| 110 | |
| 111 | func (r Reconciler) fenceClosed(ctx context.Context, obs hostclaim.Observed) (bool, error) { |
| 112 | if r.FenceClosed == nil { |
| 113 | return false, nil |
| 114 | } |
| 115 | return r.FenceClosed(ctx, obs) |
| 116 | } |
| 117 | |
| 118 | // rowDecision is §5.3's decision for this box on a fresh read. The fence |
| 119 | // observer is asked only when the decision depends on its answer, which |
| 120 | // is the source in releasing alone: every other relocating box, a target |
| 121 | // before it has a local record included, decides without it. closed is |
| 122 | // the observer's answer, false when it was not asked. |
| 123 | func (r Reconciler) rowDecision(ctx context.Context, obs hostclaim.Observed) (d RowDecision, closed bool, err error) { |
| 124 | open := RelocationRow(obs.Claim, r.Box, true, false) |
| 125 | shut := RelocationRow(obs.Claim, r.Box, true, true) |
| 126 | if open == shut { |
| 127 | return open, false, nil |
| 128 | } |
| 129 | closed, err = r.fenceClosed(ctx, obs) |
| 130 | if err != nil { |
| 131 | return RowDecision{}, false, err |
| 132 | } |
| 133 | if closed { |
| 134 | return shut, true, nil |
| 135 | } |
| 136 | return open, false, nil |
| 137 | } |
| 138 | |
| 139 | type activationReader struct { |
| 140 | r Reconciler |
| 141 | selected RecordSelection |
| 142 | } |
| 143 | |
| 144 | func (a activationReader) Read(ctx context.Context, host string) (hostclaim.Observed, error) { |
| 145 | if host != a.selected.Host { |
| 146 | return hostclaim.Observed{}, ErrChanged |
| 147 | } |
| 148 | obs, err := hostclaim.Read(ctx, a.r.Claims, host) |
| 149 | if err != nil { |
| 150 | return obs, err |
| 151 | } |
| 152 | if err := a.r.check(ctx, obs, a.selected); err != nil { |
| 153 | return hostclaim.Observed{}, err |
| 154 | } |
| 155 | return obs, nil |
| 156 | } |
| 157 | |
| 158 | // decided is Install's one row decision for a relocating claim and the |
| 159 | // path it chose: holdOnly is a kept row's hold write, with no execution. |
| 160 | type decided struct { |
| 161 | row RowDecision |
| 162 | holdOnly bool |
| 163 | } |
| 164 | |
| 165 | func (r Reconciler) check(ctx context.Context, obs hostclaim.Observed, selected RecordSelection) error { |
| 166 | return r.checkOn(ctx, obs, selected, nil) |
| 167 | } |
| 168 | |
| 169 | // checkOn is check on the row decision dec when the caller already made it |
| 170 | // (Install), or on its own fresh one (dec nil). The hold-only write is not |
| 171 | // execution, so the region's quiet window does not gate it; every other |
| 172 | // path, the activation reader's and inventory's included, is gated. |
| 173 | func (r Reconciler) checkOn(ctx context.Context, obs hostclaim.Observed, selected RecordSelection, dec *decided) error { |
| 174 | if r.Claims == nil || r.CheckRecord == nil || r.CheckLocal == nil || r.Now == nil || r.Box == "" || r.Region == "" || selected.ETag == "" || obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil { |
| 175 | return ErrUnsupported |
| 176 | } |
| 177 | h := obs.Claim.Holder |
| 178 | if obs.Host != selected.Host || h.Key != selected.Key || h.Incarnation != selected.Incarnation || h.Family != selected.Family || h.Stamp == 0 { |
| 179 | return ErrChanged |
| 180 | } |
| 181 | holdOnly := false |
| 182 | caught, catchUp, err := r.completedHold(obs.Claim) |
| 183 | if err != nil { |
| 184 | return err |
| 185 | } |
| 186 | switch { |
| 187 | case catchUp: |
| 188 | // The catch-up hold is hold only, on Install's own decision: the |
| 189 | // activation reader, the inventory callback and any caller |
| 190 | // without that decision are refused before anything is written. |
| 191 | if dec == nil || !dec.holdOnly || dec.row != caught { |
| 192 | return ErrChanged |
| 193 | } |
| 194 | holdOnly = true |
| 195 | case !r.relocates(obs.Claim): |
| 196 | if err := Serving(obs.Claim, Selection{Box: r.Box}); err != nil { |
| 197 | return err |
| 198 | } |
| 199 | default: |
| 200 | d := RowDecision{} |
| 201 | if dec != nil { |
| 202 | d, holdOnly = dec.row, dec.holdOnly |
| 203 | } else { |
| 204 | var err error |
| 205 | if d, _, err = r.rowDecision(ctx, obs); err != nil { |
| 206 | return err |
| 207 | } |
| 208 | } |
| 209 | switch d.Case { |
| 210 | case RowReady, RowHeld: |
| 211 | default: |
| 212 | return ErrChanged |
| 213 | } |
| 214 | if !holdOnly { |
| 215 | // Claim Plan 11's policy gate admits no "reconcile" action under |
| 216 | // a relocating claim (R1: only op.exec's probe), so the |
| 217 | // activation path would end refused at work.With after it had |
| 218 | // activated the guard and written the inventory. Refuse here, |
| 219 | // before anything is written; the dry run, the activation reader |
| 220 | // and the inventory callback all agree. The hold-only write |
| 221 | // takes no policy and is unchanged. |
| 222 | return ErrChanged |
| 223 | } |
| 224 | } |
| 225 | if until := park.QuietUntil(obs.Claim.QuietAfter, r.HomeRegion, r.Region); !holdOnly && !until.IsZero() && !r.Now().After(until) { |
| 226 | return ErrChanged |
| 227 | } |
| 228 | if err := r.CheckRecord(ctx, selected); err != nil { |
| 229 | return err |
| 230 | } |
| 231 | if holdOnly { |
| 232 | // Holding is not installing (Task 10b, Astra round 1): the source |
| 233 | // at installing or later is held against its own installation, |
| 234 | // which CheckHeld checks; CheckLocal would ask for the holder's |
| 235 | // stamp, the target's, and refuse for good. |
| 236 | if r.CheckHeld == nil { |
| 237 | return ErrUnsupported |
| 238 | } |
| 239 | return r.CheckHeld(ctx, obs, selected) |
| 240 | } |
| 241 | return r.CheckLocal(ctx, obs, selected) |
| 242 | } |
| 243 | func (r Reconciler) Install(ctx context.Context, selected RecordSelection, write func() error) error { |
| 244 | return r.installWithWork(ctx, selected, func(work hostwrite.Work) error { return work.With(ctx, hostwrite.Identity{}, write) }) |
| 245 | } |
| 246 | func (r Reconciler) installWithWork(ctx context.Context, selected RecordSelection, write func(hostwrite.Work) error) error { |
| 247 | if !r.Writes.Enabled { |
| 248 | if selected.DryRun { |
| 249 | return nil |
| 250 | } |
| 251 | work, err := r.Writes.Select(ctx, selected.Host, "reconcile") |
| 252 | if err != nil { |
| 253 | return err |
| 254 | } |
| 255 | return write(work) |
| 256 | } |
| 257 | if write == nil || r.Claims == nil { |
| 258 | return ErrUnsupported |
| 259 | } |
| 260 | obs, err := hostclaim.Read(ctx, r.Claims, selected.Host) |
| 261 | if err != nil { |
| 262 | return err |
| 263 | } |
| 264 | if obs.Absent { |
| 265 | if selected.DryRun { |
| 266 | return ErrUnsupported |
| 267 | } |
| 268 | if r.Adopt == nil { |
| 269 | return ErrUnsupported |
| 270 | } |
| 271 | obs, err = r.Adopt(ctx, selected) |
| 272 | if err != nil { |
| 273 | return err |
| 274 | } |
| 275 | } |
| 276 | var dec *decided |
| 277 | if r.relocates(obs.Claim) { |
| 278 | // One answer from the observer, for the check, the decision and |
| 279 | // the path, asked only when the decision depends on it. |
| 280 | d, closed, err := r.rowDecision(ctx, obs) |
| 281 | if err != nil { |
| 282 | return err |
| 283 | } |
| 284 | // No installation of ours to activate, or one whose admission |
| 285 | // fence is closed (the source in releasing, still op.exec, which |
| 286 | // With refuses): a kept row's hold only, under the flock. A |
| 287 | // target before installing has no row and mints none (ruling 17). |
| 288 | dec = &decided{row: d, holdOnly: d.Case == RowHeld && (obs.Claim.Op.Exec != r.Box || closed)} |
| 289 | } else if d, ok, err := r.completedHold(obs.Claim); err != nil { |
| 290 | return err |
| 291 | } else if ok { |
| 292 | // A completed relocation's catch-up hold: always hold only. |
| 293 | dec = &decided{row: d, holdOnly: true} |
| 294 | } |
| 295 | if err := r.checkOn(ctx, obs, selected, dec); err != nil { |
| 296 | return err |
| 297 | } |
| 298 | if selected.DryRun { |
| 299 | return nil |
| 300 | } |
| 301 | if dec != nil && dec.holdOnly { |
| 302 | return r.holdOnly(ctx, selected, dec.row) |
| 303 | } |
| 304 | if r.Prepare != nil { |
| 305 | if err := r.Prepare(ctx, obs); err != nil { |
| 306 | return err |
| 307 | } |
| 308 | } |
| 309 | // Activate's participant callback is not an admission validator for executors. |
| 310 | // The reader validates EVERY fresh read while Activate holds hostguard. |
| 311 | guard := r.Guard |
| 312 | guard.Claims = activationReader{r: r, selected: selected} |
| 313 | if err := guard.Activate(ctx, selected.Host, r.Box, nil); err != nil { |
| 314 | return err |
| 315 | } |
| 316 | captured, err := guard.Capture(selected.Host) |
| 317 | if err != nil { |
| 318 | return err |
| 319 | } |
| 320 | if r.PrepareInventory != nil { |
| 321 | if err := r.PrepareInventory(ctx, selected, captured); err != nil { |
| 322 | return err |
| 323 | } |
| 324 | } |
| 325 | selector, ok := r.Writes.Selector.(interface { |
| 326 | SelectAt(context.Context, string, string, hostguard.Captured) (hostwrite.Work, error) |
| 327 | }) |
| 328 | if !ok { |
| 329 | return ErrUnsupported |
| 330 | } |
| 331 | work, err := selector.SelectAt(ctx, selected.Host, "reconcile", captured) |
| 332 | if err != nil { |
| 333 | return err |
| 334 | } |
| 335 | work = installationWork(work) |
| 336 | local := r.Local |
| 337 | local.Guard = guard |
| 338 | check := func(c *hostclaim.Claim) error { |
| 339 | return r.check(ctx, hostclaim.Observed{Host: selected.Host, Claim: c}, selected) |
| 340 | } |
| 341 | if err := local.RecordInventory(ctx, selected.Host, captured, selected.Layout, selected.Paths, check); err != nil { |
| 342 | return err |
| 343 | } |
| 344 | return write(work) |
| 345 | } |
| 346 | |
| 347 | // holdOnly keeps an existing row held for the operation, with no |
| 348 | // installation authority: the write is conditional on the row's observed |
| 349 | // hold (registry.SetHoldIf, through RegistryHoldRow), under the host's |
| 350 | // flock. The decision is remade on a fresh read under that lock. |
| 351 | // ErrChanged when there is no row: nothing is minted. |
| 352 | func (r Reconciler) holdOnly(ctx context.Context, selected RecordSelection, d RowDecision) error { |
| 353 | if r.HoldRow == nil { |
| 354 | return ErrUnsupported |
| 355 | } |
| 356 | return r.Guard.WithLocal(ctx, selected.Host, func() error { |
| 357 | fresh, err := hostclaim.Read(ctx, r.Claims, selected.Host) |
| 358 | if err != nil { |
| 359 | return err |
| 360 | } |
| 361 | if fresh.Absent || fresh.Claim == nil || fresh.Claim.Holder == nil || fresh.Claim.Holder.Incarnation != selected.Incarnation { |
| 362 | return ErrChanged |
| 363 | } |
| 364 | // The decision is remade in whichever scope the fresh read is in: |
| 365 | // the catch-up decision (RowRetiring) and rowDecision's held one |
| 366 | // (RowHeld) never compare equal, so a scope change is ErrChanged. |
| 367 | again, catchUp, err := r.completedHold(fresh.Claim) |
| 368 | switch { |
| 369 | case err != nil: |
| 370 | return err |
| 371 | case catchUp: |
| 372 | case r.relocates(fresh.Claim): |
| 373 | if again, _, err = r.rowDecision(ctx, fresh); err != nil { |
| 374 | return err |
| 375 | } |
| 376 | default: |
| 377 | return ErrChanged |
| 378 | } |
| 379 | if again != d { |
| 380 | return ErrChanged |
| 381 | } |
| 382 | if err := r.checkOn(ctx, fresh, selected, &decided{row: d, holdOnly: true}); err != nil { |
| 383 | return err |
| 384 | } |
| 385 | return r.HoldRow(ctx, selected.Host, d.HoldOp) |
| 386 | }) |
| 387 | } |
| 388 | |
amadan