| 1 | // Package hostrelocate is the regional worker's half of relocation (region |
| 2 | // HA spec 1 §5.8, claim §4 L1): starting a relocation, and replaying every |
| 3 | // worker CAS from the claim alone. Any active enrolled box of the region |
| 4 | // may act (claim P6-D5); correctness never depends on which, because every |
| 5 | // step is a claim CAS conditional on the operation, its phase and the |
| 6 | // claim's ETag. The participants' halves (acknowledgement, release, |
| 7 | // install, probe) are not here: they run on the boxes, under the host |
| 8 | // guard, through Plan 6b's box assembly. |
| 9 | // |
| 10 | // This package only READS termination fences. hostfence.TerminationWorker |
| 11 | // writes them (control/box-fences/<enrollment>.json). |
| 12 | package hostrelocate |
| 13 | |
| 14 | import ( |
| 15 | "context" |
| 16 | "crypto/sha256" |
| 17 | "encoding/hex" |
| 18 | "encoding/json" |
| 19 | "errors" |
| 20 | "fmt" |
| 21 | "os" |
| 22 | "slices" |
| 23 | "strings" |
| 24 | "time" |
| 25 | |
| 26 | "github.com/carlosframework/platform/internal/hostactors" |
| 27 | "github.com/carlosframework/platform/internal/hostclaim" |
| 28 | "github.com/carlosframework/platform/internal/hostfence" |
| 29 | "github.com/carlosframework/platform/internal/park" |
| 30 | "github.com/carlosframework/platform/internal/release" |
| 31 | "github.com/carlosframework/platform/internal/termfence" |
| 32 | ) |
| 33 | |
| 34 | const ( |
| 35 | // ProbeDeadline is §5.8's probe deadline: a live target whose probe has |
| 36 | // not passed this long after entering probing goes to probe-failed. |
| 37 | ProbeDeadline = 10 * time.Minute |
| 38 | // ReleaseDeadline is this plan's stopping policy for a release that |
| 39 | // never completes (§5.8: "a flush that never reaches N goes to stuck"; |
| 40 | // "a probe-failed target that cannot finish its release sequence also |
| 41 | // goes to stuck"). The spec names no number; this one matches the |
| 42 | // probe deadline. A live box whose release is not journaled this long |
| 43 | // after entering releasing or probe-failed leaves the operation stuck |
| 44 | // for the operator: nothing moves away from a live box without its |
| 45 | // stop evidence. |
| 46 | ReleaseDeadline = 10 * time.Minute |
| 47 | ) |
| 48 | |
| 49 | var ( |
| 50 | ErrUnsupported = errors.New("hostrelocate: a required provider is missing") |
| 51 | ErrNotWorker = errors.New("hostrelocate: this box is not an active, unfenced worker of the region") |
| 52 | ErrNotYet = errors.New("hostrelocate: waiting for a participant or a time bound") |
| 53 | ErrRefused = errors.New("hostrelocate: relocation refused") |
| 54 | ErrNoController = errors.New("hostrelocate: no execution controller is open for this participation") |
| 55 | ) |
| 56 | |
| 57 | // Lease is the replica lease on holder.location (claim C3), as |
| 58 | // lease.Acquire returns it. The worker holds it from selecting a pin until |
| 59 | // the pin is journaled, so a park worker (which takes the same lease to |
| 60 | // retire) cannot retire the versions in between (claim §4: "New pins must |
| 61 | // be established under the required leases and fresh claim authority |
| 62 | // before conflicting retirement can proceed"). |
| 63 | type Lease interface { |
| 64 | Lost() <-chan struct{} |
| 65 | Release(context.Context) error |
| 66 | } |
| 67 | |
| 68 | // Request asks for one relocation. Target is required; Source must be the |
| 69 | // claim's executor. Reason is drain, rebalance or failover. |
| 70 | type Request struct { |
| 71 | Reason, Source, Target string |
| 72 | } |
| 73 | |
| 74 | // Owner is the app-admission registration owner of a box-started |
| 75 | // operation (claim §5.1, "A regional worker starting relocating"). |
| 76 | type Owner struct { |
| 77 | Box, Enrollment, Controller string |
| 78 | } |
| 79 | |
| 80 | // Admission is claim §5.1's app-admission registration for a box-started |
| 81 | // relocation: Register binds the saved claim observation and holder |
| 82 | // incarnation while admission is open; Recheck is the final check before |
| 83 | // the single start CAS. Claim Plan 5's app gate implements it. |
| 84 | type Admission interface { |
| 85 | Register(context.Context, hostclaim.Observed, Owner, string) (string, error) |
| 86 | Recheck(context.Context, string) error |
| 87 | } |
| 88 | |
| 89 | // Pinner selects, writes and returns one pinned recovery set (§5.8). |
| 90 | type Pinner interface { |
| 91 | Pin(context.Context, hostclaim.Observed, time.Time) (hostclaim.RecoveryPin, error) |
| 92 | } |
| 93 | |
| 94 | // Prepared verifies the remote build of the operation's prepared |
| 95 | // generation P (§5.8 "Building the prepared generation"): done is true only |
| 96 | // when a fresh read verifies P's snapshot, level-0 anchor and completed |
| 97 | // install manifest (or, for a host with nothing replicated, P's files |
| 98 | // manifest alone). Moves' shared install routine (L16) writes them. |
| 99 | type Prepared interface { |
| 100 | Verify(context.Context, hostclaim.Observed) (a hostclaim.PreparedAuthority, done bool, err error) |
| 101 | } |
| 102 | |
| 103 | // Worker drives relocation for one region. Every field is required. |
| 104 | type Worker struct { |
| 105 | Claims release.Store // the deployment (control) bucket |
| 106 | Directory hostactors.Directory |
| 107 | Fences hostclaim.FenceAuthority // hostfence.ClaimFenceAuthority |
| 108 | Releases hostclaim.ReleaseAuthority // Plan 6b's stop receipt reader |
| 109 | Admission Admission |
| 110 | // Census is the box-started start census (claim D7 with L5 and L20): |
| 111 | // hostactors.Directory.CensusBoxRelocation with the deployment's |
| 112 | // evidence and floor. |
| 113 | Census func(ctx context.Context, starter string, participants []string) error |
| 114 | // Choose names a target in the operation's region, never one in |
| 115 | // exclude (hostops.ExecutorChooser behind an adapter). |
| 116 | Choose func(ctx context.Context, obs hostclaim.Observed, exclude []string) (hostclaim.Target, error) |
| 117 | // Eligible refuses a named target that the complete eligible set |
| 118 | // (§2, boxchoice.Eligible over the roster and the edges files) does |
| 119 | // not hold: stale edges, a missing capability or secrets key, a |
| 120 | // draining box (hostops.ExecutorChooser.Permits). |
| 121 | Eligible func(ctx context.Context, obs hostclaim.Observed, box string) error |
| 122 | // Relocatable refuses a host whose authoritative record is not a |
| 123 | // relocatable shape: a unit- or socket-backed route, a tunnel, a |
| 124 | // sidecar (the same shapes Plan 6b's wake refuses). |
| 125 | Relocatable func(ctx context.Context, obs hostclaim.Observed) error |
| 126 | // Leases acquires the replica lease on the host's location in the |
| 127 | // region's data bucket (lease.Acquire). |
| 128 | Leases func(ctx context.Context, location string) (Lease, error) |
| 129 | Pins Pinner |
| 130 | Prepared Prepared |
| 131 | // Discharge judges a fenced box's volume records (boxvolumes.Discharge). |
| 132 | // It never gates anything: nil, or an unqualified verdict, leaves the |
| 133 | // box's park-local chores open for an operator (H19). |
| 134 | Discharge func(context.Context, hostactors.Box, termfence.Record) *hostclaim.LocalDischarge |
| 135 | // Notify bumps the control serial after an executor CAS (§5.3, |
| 136 | // "Prompt changes"). A failed bump costs cadence, never correctness. |
| 137 | Notify func(context.Context) |
| 138 | Self termfence.WrittenBy |
| 139 | Region string |
| 140 | // HomeRegion is this box's verified home binding (hostbox.Actor's, the |
| 141 | // fleet's home roster name when Region is it), or "" with no alias. It |
| 142 | // is the one binding under which the home's two names, Region and |
| 143 | // park.HomeRegion, are one region (park.SameRegion), as main's claim |
| 144 | // owners read it (park.VerifiedHome against the op's "home_region"). |
| 145 | HomeRegion string |
| 146 | Controller string |
| 147 | NewID func() string |
| 148 | Now func() time.Time |
| 149 | } |
| 150 | |
| 151 | func (w Worker) ready() error { |
| 152 | if w.Claims == nil || w.Directory.Store == nil || w.Fences == nil || w.Releases == nil || w.Admission == nil || w.Census == nil || |
| 153 | w.Choose == nil || w.Eligible == nil || w.Relocatable == nil || w.Leases == nil || w.Pins == nil || w.Prepared == nil || w.Discharge == nil || w.Notify == nil || w.Self.Box == "" || w.Self.Enrollment == "" || |
| 154 | w.Region == "" || w.Controller == "" || w.NewID == nil || w.Now == nil { |
| 155 | return ErrUnsupported |
| 156 | } |
| 157 | return nil |
| 158 | } |
| 159 | |
| 160 | // self checks, on every call, that this box is an active, unfenced worker |
| 161 | // enrolled in the region (§5.8 "Each worker first checks its own roster |
| 162 | // entry is active in this region"). |
| 163 | func (w Worker) self(ctx context.Context) (hostactors.Observed, error) { |
| 164 | o, err := w.Directory.Read(ctx) |
| 165 | if err != nil { |
| 166 | return hostactors.Observed{}, err |
| 167 | } |
| 168 | b, ok := o.Roster.Boxes[w.Self.Box] |
| 169 | if !ok || b.State != "active" || b.Enrollment != w.Self.Enrollment || b.Region != w.Region { |
| 170 | return hostactors.Observed{}, ErrNotWorker |
| 171 | } |
| 172 | // A home binding is the signed roster's home, and only the home's own |
| 173 | // boxes hold it (hostbox.IdentityProvider.Verify's rule). |
| 174 | if w.HomeRegion != "" && (w.HomeRegion != o.Roster.HomeRegion || w.HomeRegion != w.Region) { |
| 175 | return hostactors.Observed{}, ErrNotWorker |
| 176 | } |
| 177 | fenced, err := w.Directory.TerminationFence(ctx, o, w.Self.Box, w.Now()) |
| 178 | if err != nil || fenced { |
| 179 | return hostactors.Observed{}, errors.Join(ErrNotWorker, err) |
| 180 | } |
| 181 | return o, nil |
| 182 | } |
| 183 | |
| 184 | // fence reads box's termination fence, if any, as the exact bytes a claim |
| 185 | // CAS will cite. Absence is never evidence: no fence is (nil, nil); an |
| 186 | // unreadable or invalid one is an error, never "not fenced". |
| 187 | func (w Worker) fence(ctx context.Context, o hostactors.Observed, box string) (*hostclaim.FenceRef, termfence.Record, error) { |
| 188 | b, ok := o.Roster.Boxes[box] |
| 189 | if !ok { |
| 190 | return nil, termfence.Record{}, fmt.Errorf("%w: %s is not in the roster", ErrRefused, box) |
| 191 | } |
| 192 | key := hostfence.FenceKey(b.Enrollment) |
| 193 | raw, _, err := w.Directory.Store.Get(ctx, key) |
| 194 | if errors.Is(err, os.ErrNotExist) { |
| 195 | return nil, termfence.Record{}, nil |
| 196 | } |
| 197 | if err != nil { |
| 198 | return nil, termfence.Record{}, err |
| 199 | } |
| 200 | var f termfence.Record |
| 201 | if err := json.Unmarshal(raw, &f); err != nil { |
| 202 | return nil, termfence.Record{}, err |
| 203 | } |
| 204 | if err := hostactors.ValidateTerminationFence(o, box, f, w.Now()); err != nil { |
| 205 | return nil, termfence.Record{}, err |
| 206 | } |
| 207 | sum := sha256.Sum256(raw) |
| 208 | return &hostclaim.FenceRef{Enrollment: b.Enrollment, Key: key, SHA256: hex.EncodeToString(sum[:])}, f, nil |
| 209 | } |
| 210 | |
| 211 | // ConvertLanding is J3 for one landing claim, under the same worker |
| 212 | // authorization as Step (Astra round 6 finding 4): this box must be an |
| 213 | // active, unfenced worker whose current roster entry is the enrollment it |
| 214 | // runs as, in the region the landing lands in. A landing whose op.exec |
| 215 | // box (the move's target) has a termination fence becomes a failover |
| 216 | // relocation when its move committed (guard); an uncommitted one is |
| 217 | // moves' case (ErrWrongState). A landing into another region, one with no |
| 218 | // executor, or one whose target has no fence is left alone with nothing |
| 219 | // written; a caller that is not such a worker is ErrNotWorker with nothing |
| 220 | // written. |
| 221 | func (w Worker) ConvertLanding(ctx context.Context, obs hostclaim.Observed, guard hostclaim.CommittedLandingGuard) (hostclaim.Observed, error) { |
| 222 | if w.Claims == nil || w.Directory.Store == nil || w.Fences == nil || w.Discharge == nil || w.NewID == nil || w.Now == nil || w.Self.Box == "" || w.Self.Enrollment == "" || w.Region == "" || guard == nil { |
| 223 | return obs, ErrUnsupported |
| 224 | } |
| 225 | o, err := w.self(ctx) |
| 226 | if err != nil { |
| 227 | return obs, err |
| 228 | } |
| 229 | if obs.Absent || obs.Claim == nil { |
| 230 | return obs, nil |
| 231 | } |
| 232 | c := obs.Claim |
| 233 | box := c.Op.Exec |
| 234 | if c.Op.Kind != hostclaim.KindLanding || box == "" || box == "none" { |
| 235 | return obs, nil |
| 236 | } |
| 237 | // The landing's "to" is an engine region (park.HomeRegion for the |
| 238 | // home); this worker's Region is its roster name. They are one region |
| 239 | // only under a home binding this box and the move agree on. |
| 240 | home, err := park.VerifiedHome(o.Roster.HomeRegion, w.HomeRegion, w.Region, c.Op.Intent["home_region"]) |
| 241 | if err != nil { |
| 242 | return obs, err |
| 243 | } |
| 244 | if !park.SameRegion(home, c.Op.Intent["to"], w.Region) { |
| 245 | return obs, nil |
| 246 | } |
| 247 | ref, record, err := w.fence(ctx, o, box) |
| 248 | if err != nil || ref == nil { |
| 249 | return obs, err |
| 250 | } |
| 251 | d := w.Discharge(ctx, o.Roster.Boxes[box], record) |
| 252 | return hostclaim.ConvertCommittedLanding(ctx, w.Claims, obs, c.Op.ID, box, *ref, w.Fences, guard, d, w.NewID) |
| 253 | } |
| 254 | |
| 255 | // newestFenceObservation is the latest fence.<box>.observed_at the op |
| 256 | // journals. An observation that does not parse, or none at all, is an |
| 257 | // error, never "no bound" (absence is never evidence): the conversion |
| 258 | // journals its target's fence observation in the same CAS that leaves the |
| 259 | // op waiting, so a wait with nothing to bound it is a defect, refused as |
| 260 | // an unreadable installed_at is. |
| 261 | func newestFenceObservation(c *hostclaim.Claim) (time.Time, error) { |
| 262 | var newest time.Time |
| 263 | for k, v := range c.Op.Journal { |
| 264 | if !strings.HasPrefix(k, "fence.") || !strings.HasSuffix(k, ".observed_at") { |
| 265 | continue |
| 266 | } |
| 267 | at, err := time.Parse(time.RFC3339Nano, v) |
| 268 | if err != nil { |
| 269 | return time.Time{}, fmt.Errorf("%w: %s of %s: %v", ErrRefused, k, c.Op.ID, err) |
| 270 | } |
| 271 | if at.After(newest) { |
| 272 | newest = at |
| 273 | } |
| 274 | } |
| 275 | if newest.IsZero() { |
| 276 | return time.Time{}, fmt.Errorf("%w: %s journals no fence observation to bound its wait for a target", ErrRefused, c.Op.ID) |
| 277 | } |
| 278 | return newest, nil |
| 279 | } |
| 280 | |
| 281 | // Start writes the start CAS for req (§5.8 "The start CAS", §5.10). It |
| 282 | // refuses unless the claim serves on req.Source; the roster shows the |
| 283 | // starter, the target and (for a drain) the source active, unfenced and |
| 284 | // with no executable update reserved (H14); the census passes; and the |
| 285 | // app admits the registration. A failover requires the source's fence. |
| 286 | func (w Worker) Start(ctx context.Context, host string, req Request) (hostclaim.Observed, error) { |
| 287 | if err := w.ready(); err != nil { |
| 288 | return hostclaim.Observed{}, err |
| 289 | } |
| 290 | o, err := w.self(ctx) |
| 291 | if err != nil { |
| 292 | return hostclaim.Observed{}, err |
| 293 | } |
| 294 | obs, err := hostclaim.Read(ctx, w.Claims, host) |
| 295 | if err != nil { |
| 296 | return hostclaim.Observed{}, err |
| 297 | } |
| 298 | if obs.Absent || obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Exec != req.Source || req.Target == req.Source { |
| 299 | return hostclaim.Observed{}, fmt.Errorf("%w: %s does not serve on %s", ErrRefused, host, req.Source) |
| 300 | } |
| 301 | if obs.Claim.Holder.Family != "instance" { |
| 302 | return hostclaim.Observed{}, fmt.Errorf("%w: a %s host is not relocatable", ErrRefused, obs.Claim.Holder.Family) |
| 303 | } |
| 304 | if err := w.Relocatable(ctx, obs); err != nil { |
| 305 | return hostclaim.Observed{}, err |
| 306 | } |
| 307 | if err := w.Eligible(ctx, obs, req.Target); err != nil { |
| 308 | return hostclaim.Observed{}, err |
| 309 | } |
| 310 | source, sok := o.Roster.Boxes[req.Source] |
| 311 | target, tok := o.Roster.Boxes[req.Target] |
| 312 | if !sok || !tok || source.Region != w.Region || target.Region != w.Region { |
| 313 | return hostclaim.Observed{}, fmt.Errorf("%w: source and target must be enrolled in %s", ErrRefused, w.Region) |
| 314 | } |
| 315 | ref, record, err := w.fence(ctx, o, req.Source) |
| 316 | if err != nil { |
| 317 | return hostclaim.Observed{}, err |
| 318 | } |
| 319 | refs := []string{w.Self.Box, req.Target} |
| 320 | switch req.Reason { |
| 321 | case hostclaim.ReasonFailover: |
| 322 | if ref == nil { |
| 323 | return hostclaim.Observed{}, fmt.Errorf("%w: a failover needs the source's termination fence", ErrRefused) |
| 324 | } |
| 325 | case hostclaim.ReasonDrain, hostclaim.ReasonRebalance: |
| 326 | if ref != nil { |
| 327 | return hostclaim.Observed{}, fmt.Errorf("%w: the source is fenced; start a failover", ErrRefused) |
| 328 | } |
| 329 | if obs.Claim.Holder.Session == 0 { |
| 330 | // Ruling 11 (rev 2): a never-woken source has no writer session |
| 331 | // to stop, and root refuses stop evidence for session 0 until |
| 332 | // claim Plan 5's no-process witness lands. Until then only a |
| 333 | // failover relocates such a host. |
| 334 | return hostclaim.Observed{}, fmt.Errorf("%w: %s never admitted a writer on %s; a drain has nothing to release", ErrRefused, host, req.Source) |
| 335 | } |
| 336 | refs = append(refs, req.Source) |
| 337 | default: |
| 338 | return hostclaim.Observed{}, fmt.Errorf("%w: reason %q", ErrRefused, req.Reason) |
| 339 | } |
| 340 | refs = slices.Compact(slices.Sorted(slices.Values(refs))) |
| 341 | revision, err := w.Directory.CheckBoxStart(ctx, refs...) |
| 342 | if err != nil { |
| 343 | return hostclaim.Observed{}, err |
| 344 | } |
| 345 | if err := w.Census(ctx, w.Self.Box, refs); err != nil { |
| 346 | return hostclaim.Observed{}, err |
| 347 | } |
| 348 | id := w.NewID() |
| 349 | admission, err := w.Admission.Register(ctx, obs, Owner{Box: w.Self.Box, Enrollment: w.Self.Enrollment, Controller: w.Controller}, id) |
| 350 | if err != nil { |
| 351 | return hostclaim.Observed{}, err |
| 352 | } |
| 353 | if err := w.Admission.Recheck(ctx, admission); err != nil { |
| 354 | return hostclaim.Observed{}, err |
| 355 | } |
| 356 | in := hostclaim.RelocationStart{OpID: id, By: "box:" + w.Self.Box, Now: w.Now(), Region: w.Region, HomeRegion: w.HomeRegion, Reason: req.Reason, |
| 357 | SourceBox: req.Source, SourceEnrollment: source.Enrollment, TargetBox: req.Target, TargetEnrollment: target.Enrollment, |
| 358 | DirectoryRevision: revision, AdmissionID: admission, Fence: ref} |
| 359 | if ref != nil { |
| 360 | in.Discharge = w.Discharge(ctx, source, record) |
| 361 | } |
| 362 | next, err := hostclaim.StartRelocation(ctx, w.Claims, obs, in, w.Fences) |
| 363 | if err == nil && ref != nil { |
| 364 | w.Notify(ctx) |
| 365 | } |
| 366 | return next, err |
| 367 | } |
| 368 | |
| 369 | // Step makes at most one worker CAS for host's relocation, the one its |
| 370 | // phase and evidence call for, and returns the claim it left. ErrNotYet |
| 371 | // means a participant or a time bound is awaited; nothing was written. |
| 372 | // A host that is not relocating, or whose operation is stuck, is left |
| 373 | // alone. |
| 374 | func (w Worker) Step(ctx context.Context, host string) (hostclaim.Observed, error) { |
| 375 | if err := w.ready(); err != nil { |
| 376 | return hostclaim.Observed{}, err |
| 377 | } |
| 378 | o, err := w.self(ctx) |
| 379 | if err != nil { |
| 380 | return hostclaim.Observed{}, err |
| 381 | } |
| 382 | obs, err := hostclaim.Read(ctx, w.Claims, host) |
| 383 | if err != nil || obs.Absent { |
| 384 | return obs, err |
| 385 | } |
| 386 | c := obs.Claim |
| 387 | if c.Op.Kind != hostclaim.KindRelocating || c.Op.Intent["region"] != w.Region || c.Op.Journal["stuck"] != "" { |
| 388 | return obs, nil |
| 389 | } |
| 390 | // An op bound to a home names it in "home_region"; this box steps it |
| 391 | // only under the same binding (park.VerifiedHome, main's rule). |
| 392 | if _, err := park.VerifiedHome(o.Roster.HomeRegion, w.HomeRegion, w.Region, c.Op.Intent["home_region"]); err != nil { |
| 393 | return obs, err |
| 394 | } |
| 395 | id := c.Op.ID |
| 396 | // Evidence first: a fenced participant changes what every phase may do. |
| 397 | // A pass that refuses here checks the release deadline first (refused). |
| 398 | if src := c.Op.Intent["source_box"]; !slices.Contains(c.Lost, src) { |
| 399 | ref, record, err := w.fence(ctx, o, src) |
| 400 | if err != nil { |
| 401 | return w.refused(ctx, obs, err) |
| 402 | } |
| 403 | if ref == nil && c.Op.Journal["fence."+src] != "" { |
| 404 | // A converted move bound the source's fence before FenceSource |
| 405 | // consumed it. A bound fence that is no longer there is not |
| 406 | // "not fenced" (absence is never evidence): refuse the pass. |
| 407 | return w.refused(ctx, obs, fmt.Errorf("%w: %s's fence is bound in the journal but cannot be read", ErrRefused, src)) |
| 408 | } |
| 409 | if ref != nil { |
| 410 | next, err := hostclaim.FenceSource(ctx, w.Claims, obs, id, *ref, w.Discharge(ctx, o.Roster.Boxes[src], record), w.Fences) |
| 411 | if err == nil { |
| 412 | w.Notify(ctx) |
| 413 | } |
| 414 | return next, err |
| 415 | } |
| 416 | } |
| 417 | if dst := c.Op.Intent["target_box"]; dst != "" && !slices.Contains(c.Lost, dst) { |
| 418 | ref, record, err := w.fence(ctx, o, dst) |
| 419 | if err != nil { |
| 420 | return w.refused(ctx, obs, err) |
| 421 | } |
| 422 | if ref != nil { |
| 423 | return w.replace(ctx, obs, *ref, w.Discharge(ctx, o.Roster.Boxes[dst], record), record.ObservedAt) |
| 424 | } |
| 425 | } |
| 426 | switch c.Op.Phase { |
| 427 | case hostclaim.PhaseStarting: |
| 428 | if c.Op.Intent["target_box"] == "" && c.Op.Exec == "none" { |
| 429 | // A committed landing converted to a failover |
| 430 | // (hostclaim.ConvertCommittedLanding) starts with no target: name |
| 431 | // one here, so it acknowledges before PinFailover selects a pin, |
| 432 | // as an initial failover's target does. |
| 433 | next, err := w.next(ctx, obs) |
| 434 | if err != nil { |
| 435 | // J3-3: bounded. A converted landing with no eligible |
| 436 | // target is stuck ReleaseDeadline after its target's |
| 437 | // fence was observed; before that the chooser's refusal |
| 438 | // is returned and the pass alerts once. |
| 439 | at, ferr := newestFenceObservation(c) |
| 440 | if ferr != nil { |
| 441 | return obs, errors.Join(err, ferr) |
| 442 | } |
| 443 | if w.Now().Sub(at) > ReleaseDeadline { |
| 444 | return w.update(ctx, obs, hostclaim.MarkStuck(id, fmt.Sprintf("no eligible box for the converted landing after %s: %v", ReleaseDeadline, err))) |
| 445 | } |
| 446 | return obs, err |
| 447 | } |
| 448 | return w.update(ctx, obs, hostclaim.NameTarget(id, next)) |
| 449 | } |
| 450 | if !acked(c) { |
| 451 | return obs, ErrNotYet |
| 452 | } |
| 453 | if c.Op.Exec == "none" { |
| 454 | return w.update(ctx, obs, hostclaim.PinFailover(id)) |
| 455 | } |
| 456 | return w.update(ctx, obs, hostclaim.ToReleasing(id, w.Now())) |
| 457 | case hostclaim.PhaseReleasing: |
| 458 | if c.Op.Journal["release"] == "" { |
| 459 | return w.releaseOverdue(ctx, obs, "releasing_since") |
| 460 | } |
| 461 | return hostclaim.PinAfterRelease(ctx, w.Claims, obs, id, w.Releases, nil) |
| 462 | case hostclaim.PhaseProbeFailed: |
| 463 | if c.Op.Journal["release"] == "" { |
| 464 | return w.releaseOverdue(ctx, obs, "probe_failed_since") |
| 465 | } |
| 466 | if done, err := exhausted(c); err != nil { |
| 467 | return obs, err |
| 468 | } else if done { |
| 469 | return w.stuckOn(ctx, obs, hostclaim.ErrReplacementsExceeded) |
| 470 | } |
| 471 | next, err := w.next(ctx, obs) |
| 472 | if err != nil { |
| 473 | // No eligible box yet; a later pass may find one, but not for |
| 474 | // ever: in a two-box region the source is excluded and there |
| 475 | // may be none (rev 17, finding F1). ReleaseDeadline after the |
| 476 | // target failed, the operation is stuck for an operator. |
| 477 | since, perr := time.Parse(time.RFC3339Nano, c.Op.Journal["probe_failed_since"]) |
| 478 | if perr == nil && w.Now().Sub(since) > ReleaseDeadline { |
| 479 | return w.update(ctx, obs, hostclaim.MarkStuck(id, fmt.Sprintf("no eligible box for the nested relocation after %s: %v", ReleaseDeadline, err))) |
| 480 | } |
| 481 | return obs, err |
| 482 | } |
| 483 | out, err := hostclaim.PinAfterRelease(ctx, w.Claims, obs, id, w.Releases, &next) |
| 484 | if err != nil { |
| 485 | // A live target's replacement counts too: the third is stuck. |
| 486 | return w.stuckOn(ctx, obs, err) |
| 487 | } |
| 488 | return out, nil |
| 489 | case hostclaim.PhasePinning: |
| 490 | return w.pinning(ctx, obs) |
| 491 | case hostclaim.PhaseRelocatingProbing: |
| 492 | since, err := time.Parse(time.RFC3339Nano, c.Op.Journal["probing_since"]) |
| 493 | if err != nil { |
| 494 | return obs, err |
| 495 | } |
| 496 | if c.Op.Journal["probe"] != "" && c.Op.Journal["probe_stop"] != "" { |
| 497 | next, err := w.update(ctx, obs, hostclaim.CompleteRelocation(id, w.NewID)) |
| 498 | if err == nil { |
| 499 | w.Notify(ctx) |
| 500 | } |
| 501 | return next, err |
| 502 | } |
| 503 | if c.Op.Journal["probe"] != "" { |
| 504 | // The target stops its probe session first (ruling 49). Ruling |
| 505 | // 67: bounded. If root's stop receipt cannot be had (the |
| 506 | // execution lease lost after RecordProbe, so the participant |
| 507 | // never passes Ready; a stop root keeps refusing), the op is |
| 508 | // stuck ReleaseDeadline after the probe passed (Probe.At; for an |
| 509 | // untimed probe journaled before rev 4, probing_since plus |
| 510 | // ProbeDeadline, the latest it can have passed), for an |
| 511 | // operator. Nothing is released: the participation's denial, |
| 512 | // the row's hold and the session stay as they are. |
| 513 | var p hostclaim.Probe |
| 514 | passed := since.Add(ProbeDeadline) |
| 515 | if json.Unmarshal([]byte(c.Op.Journal["probe"]), &p) == nil && !p.At.IsZero() { |
| 516 | passed = p.At |
| 517 | } |
| 518 | if w.Now().Sub(passed) > ReleaseDeadline { |
| 519 | return w.update(ctx, obs, hostclaim.MarkStuck(id, fmt.Sprintf("%s passed its probe but no root stop receipt for session %d within %s: an operator must obtain the stop", c.Op.Exec, p.Session, ReleaseDeadline))) |
| 520 | } |
| 521 | return obs, ErrNotYet |
| 522 | } |
| 523 | if w.Now().Sub(since) > ProbeDeadline { |
| 524 | return w.update(ctx, obs, hostclaim.ToProbeFailed(id, w.Now())) |
| 525 | } |
| 526 | return obs, ErrNotYet |
| 527 | case hostclaim.PhaseInstalling: |
| 528 | a, done, err := w.Prepared.Verify(ctx, obs) |
| 529 | if err != nil || !done || !acked(c) { |
| 530 | return obs, errors.Join(ErrNotYet, err) |
| 531 | } |
| 532 | return w.update(ctx, obs, hostclaim.ToInstalledAt(id, a, w.Now())) |
| 533 | case hostclaim.PhaseInstalled: |
| 534 | // The target's local install and its entry into probing (Plan 6b). |
| 535 | // Ruling 60: bounded. A target that cannot enter probing within |
| 536 | // ProbeDeadline (a retired stamp, a participation main denies) is |
| 537 | // stuck for an operator: no automatic replacement of a live target |
| 538 | // is made here. |
| 539 | // A missing or unreadable installed_at is an error, never "no |
| 540 | // bound" (absence is never evidence), as probing_since is above. |
| 541 | at, err := time.Parse(time.RFC3339Nano, c.Op.Journal["installed_at"]) |
| 542 | if err != nil { |
| 543 | return obs, fmt.Errorf("installed_at of %s: %w", id, err) |
| 544 | } |
| 545 | if w.Now().Sub(at) > ProbeDeadline { |
| 546 | return w.update(ctx, obs, hostclaim.MarkStuck(id, fmt.Sprintf("%s did not enter probing within %s of installed", c.Op.Exec, ProbeDeadline))) |
| 547 | } |
| 548 | return obs, ErrNotYet |
| 549 | } |
| 550 | return obs, ErrNotYet |
| 551 | } |
| 552 | |
| 553 | func (w Worker) pinning(ctx context.Context, obs hostclaim.Observed) (hostclaim.Observed, error) { |
| 554 | c := obs.Claim |
| 555 | id := c.Op.ID |
| 556 | if c.Op.Intent["target_box"] == "" { |
| 557 | next, err := w.next(ctx, obs) |
| 558 | if err != nil { |
| 559 | return obs, err |
| 560 | } |
| 561 | return w.update(ctx, obs, hostclaim.NameTarget(id, next)) |
| 562 | } |
| 563 | if c.Op.Journal["pin"] == "" { |
| 564 | for key, at := range c.Op.Journal { |
| 565 | if len(key) > len(".observed_at") && key[len(key)-len(".observed_at"):] == ".observed_at" { |
| 566 | t, err := time.Parse(time.RFC3339Nano, at) |
| 567 | if err != nil { |
| 568 | return obs, err |
| 569 | } |
| 570 | if w.Now().Before(t.Add(hostclaim.FailoverPinDelay)) { |
| 571 | return obs, ErrNotYet |
| 572 | } |
| 573 | } |
| 574 | } |
| 575 | return w.pinUnderLease(ctx, obs) |
| 576 | } |
| 577 | t := hostclaim.Target{Box: c.Op.Intent["target_box"], Enrollment: c.Op.Intent["target_enrollment"]} |
| 578 | if !acked(c) { |
| 579 | return obs, ErrNotYet |
| 580 | } |
| 581 | next, err := w.update(ctx, obs, hostclaim.ToInstalling(id, t, w.NewID())) |
| 582 | if err == nil { |
| 583 | w.Notify(ctx) |
| 584 | } |
| 585 | return next, err |
| 586 | } |
| 587 | |
| 588 | // pinUnderLease selects and journals the pin while holding the replica |
| 589 | // lease on holder.location, so no park can retire the pinned versions |
| 590 | // between the selection and the journal CAS (claim §4). A lease lost |
| 591 | // before the CAS refuses: the next pass selects afresh. |
| 592 | func (w Worker) pinUnderLease(ctx context.Context, obs hostclaim.Observed) (hostclaim.Observed, error) { |
| 593 | held, err := w.Leases(ctx, obs.Claim.Holder.Location) |
| 594 | if err != nil { |
| 595 | return obs, err |
| 596 | } |
| 597 | defer func() { _ = held.Release(ctx) }() |
| 598 | pin, err := w.Pins.Pin(ctx, obs, w.Now()) |
| 599 | if errors.Is(err, ErrFilesUnevidenced) { |
| 600 | // H6: instance.key, report.token and the env are continuous; a pin |
| 601 | // that cannot show them never existed waits for an operator. |
| 602 | return w.update(ctx, obs, hostclaim.MarkStuck(obs.Claim.Op.ID, err.Error())) |
| 603 | } |
| 604 | if err != nil { |
| 605 | return obs, err |
| 606 | } |
| 607 | select { |
| 608 | case <-held.Lost(): |
| 609 | return obs, fmt.Errorf("%w: the location lease was lost before the pin was journaled", ErrNotYet) |
| 610 | default: |
| 611 | } |
| 612 | return w.update(ctx, obs, hostclaim.JournalPin(obs.Claim.Op.ID, pin)) |
| 613 | } |
| 614 | |
| 615 | // refused is a pass that refuses with cause. The release deadline is |
| 616 | // checked before the refusal stands: in releasing or probe-failed, |
| 617 | // ReleaseDeadline after the phase began (ruling 14), the operation is |
| 618 | // stuck instead, whether or not a release was journaled. So a refusal |
| 619 | // that never clears, a source removed from the roster before its pin |
| 620 | // (Task 7's ruling), leaves the operation stuck for an operator, never |
| 621 | // refusing for ever. Only refusals (ErrRefused) count: any other error is |
| 622 | // returned as it is, for a later pass to retry. |
| 623 | func (w Worker) refused(ctx context.Context, obs hostclaim.Observed, cause error) (hostclaim.Observed, error) { |
| 624 | sinceKey := map[string]string{hostclaim.PhaseReleasing: "releasing_since", hostclaim.PhaseProbeFailed: "probe_failed_since"}[obs.Claim.Op.Phase] |
| 625 | if sinceKey == "" || !errors.Is(cause, ErrRefused) { |
| 626 | return obs, cause |
| 627 | } |
| 628 | since, err := time.Parse(time.RFC3339Nano, obs.Claim.Op.Journal[sinceKey]) |
| 629 | if err != nil { |
| 630 | return obs, errors.Join(cause, err) |
| 631 | } |
| 632 | if w.Now().Sub(since) <= ReleaseDeadline { |
| 633 | return obs, cause |
| 634 | } |
| 635 | return w.update(ctx, obs, hostclaim.MarkStuck(obs.Claim.Op.ID, fmt.Sprintf("%s refused past the release deadline (%s): %v", obs.Claim.Op.Phase, ReleaseDeadline, cause))) |
| 636 | } |
| 637 | |
| 638 | // releaseOverdue marks the operation stuck when a live box has not |
| 639 | // journaled its release ReleaseDeadline after entering the phase whose |
| 640 | // start key names; before that it waits. |
| 641 | func (w Worker) releaseOverdue(ctx context.Context, obs hostclaim.Observed, sinceKey string) (hostclaim.Observed, error) { |
| 642 | since, err := time.Parse(time.RFC3339Nano, obs.Claim.Op.Journal[sinceKey]) |
| 643 | if err != nil { |
| 644 | return obs, err |
| 645 | } |
| 646 | if w.Now().Sub(since) <= ReleaseDeadline { |
| 647 | return obs, ErrNotYet |
| 648 | } |
| 649 | return w.update(ctx, obs, hostclaim.MarkStuck(obs.Claim.Op.ID, fmt.Sprintf("%s has not released %s after %s", obs.Claim.Op.Exec, ReleaseDeadline, obs.Claim.Op.Phase))) |
| 650 | } |
| 651 | |
| 652 | // next names the operation's next target: never the source, a lost box, |
| 653 | // a box this operation already replaced, or a box that owes a park-local |
| 654 | // chore for the host (spec 1 §2; ReplaceTarget refuses one in its CAS). |
| 655 | func (w Worker) next(ctx context.Context, obs hostclaim.Observed) (hostclaim.Target, error) { |
| 656 | c := obs.Claim |
| 657 | exclude := append([]string{c.Op.Intent["source_box"]}, c.Lost...) |
| 658 | exclude = append(exclude, hostclaim.LocalChoreBoxes(c)...) |
| 659 | if t := c.Op.Intent["target_box"]; t != "" { |
| 660 | exclude = append(exclude, t) |
| 661 | } |
| 662 | var replaced []string |
| 663 | if raw := c.Op.Journal["excluded_targets"]; raw != "" { |
| 664 | if err := json.Unmarshal([]byte(raw), &replaced); err != nil { |
| 665 | return hostclaim.Target{}, err |
| 666 | } |
| 667 | } |
| 668 | return w.Choose(ctx, obs, append(exclude, replaced...)) |
| 669 | } |
| 670 | |
| 671 | func (w Worker) replace(ctx context.Context, obs hostclaim.Observed, ref hostclaim.FenceRef, d *hostclaim.LocalDischarge, observedAt string) (hostclaim.Observed, error) { |
| 672 | // Exhaustion is checked before a box is chosen (§5.8 "The bound"): |
| 673 | // with the replacements used, no spare is looked for; the dead |
| 674 | // target's fence is consumed into a stuck pinning (the loss retained, |
| 675 | // its installation's authority revoked, no probe to wait for). |
| 676 | done, err := exhausted(obs.Claim) |
| 677 | if err != nil { |
| 678 | return obs, err |
| 679 | } |
| 680 | if done { |
| 681 | out, err := hostclaim.FenceExhaustedTarget(ctx, w.Claims, obs, obs.Claim.Op.ID, ref, d, w.Fences) |
| 682 | if err == nil { |
| 683 | w.Notify(ctx) |
| 684 | } |
| 685 | return out, err |
| 686 | } |
| 687 | next, err := w.next(ctx, obs) |
| 688 | if err != nil { |
| 689 | // No eligible box to replace the dead target. A later pass may |
| 690 | // find one, but not for ever (rev 18, round 17 finding 2): |
| 691 | // ReleaseDeadline after the fence's observation the fence is |
| 692 | // consumed and the operation is stuck for an operator. |
| 693 | at, perr := time.Parse(time.RFC3339Nano, observedAt) |
| 694 | if perr == nil && w.Now().Sub(at) > ReleaseDeadline { |
| 695 | out, ferr := hostclaim.FenceTargetWithoutSpare(ctx, w.Claims, obs, obs.Claim.Op.ID, ref, d, w.Fences, |
| 696 | fmt.Sprintf("%s terminated and no eligible box replaced it after %s: %v", obs.Claim.Op.Intent["target_box"], ReleaseDeadline, err)) |
| 697 | if ferr == nil { |
| 698 | w.Notify(ctx) |
| 699 | } |
| 700 | return out, ferr |
| 701 | } |
| 702 | return obs, err |
| 703 | } |
| 704 | out, err := hostclaim.ReplaceTarget(ctx, w.Claims, obs, obs.Claim.Op.ID, ref, d, next, w.NewID(), w.Fences) |
| 705 | if errors.Is(err, hostclaim.ErrReplacementsExceeded) { |
| 706 | return w.stuckOn(ctx, obs, err) |
| 707 | } |
| 708 | if err == nil { |
| 709 | w.Notify(ctx) |
| 710 | } |
| 711 | return out, err |
| 712 | } |
| 713 | |
| 714 | // stuckOn marks the operation stuck when it has used its replacements; |
| 715 | // any other error is returned for a later pass to retry. |
| 716 | func (w Worker) stuckOn(ctx context.Context, obs hostclaim.Observed, cause error) (hostclaim.Observed, error) { |
| 717 | if !errors.Is(cause, hostclaim.ErrReplacementsExceeded) { |
| 718 | return obs, cause |
| 719 | } |
| 720 | return w.update(ctx, obs, hostclaim.MarkStuck(obs.Claim.Op.ID, cause.Error())) |
| 721 | } |
| 722 | |
| 723 | func (w Worker) update(ctx context.Context, obs hostclaim.Observed, tr func(*hostclaim.Claim) error) (hostclaim.Observed, error) { |
| 724 | return hostclaim.Update(ctx, w.Claims, obs, tr) |
| 725 | } |
| 726 | |
| 727 | // exhausted reports whether the operation has used its target |
| 728 | // replacements. A malformed count is an error, never 0. |
| 729 | func exhausted(c *hostclaim.Claim) (bool, error) { |
| 730 | n, err := hostclaim.Replacements(c) |
| 731 | if err != nil { |
| 732 | return false, err |
| 733 | } |
| 734 | return n >= hostclaim.MaxTargetReplacements, nil |
| 735 | } |
| 736 | |
| 737 | // acked is the D6 barrier as the worker reads it before a CAS that needs it. |
| 738 | func acked(c *hostclaim.Claim) bool { |
| 739 | for _, box := range c.Op.Participants { |
| 740 | if c.Op.Journal["exec_ack."+box] == "" { |
| 741 | return false |
| 742 | } |
| 743 | } |
| 744 | return true |
| 745 | } |
| 746 | |
amadan