| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/sha256" |
| 7 | "encoding/hex" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "fmt" |
| 11 | "io/fs" |
| 12 | "log" |
| 13 | "os" |
| 14 | "path/filepath" |
| 15 | "reflect" |
| 16 | "strings" |
| 17 | "sync" |
| 18 | "time" |
| 19 | |
| 20 | "github.com/carlosframework/platform/internal/activator" |
| 21 | "github.com/carlosframework/platform/internal/appconfig" |
| 22 | "github.com/carlosframework/platform/internal/genselect" |
| 23 | "github.com/carlosframework/platform/internal/hostbox" |
| 24 | "github.com/carlosframework/platform/internal/hostclaim" |
| 25 | "github.com/carlosframework/platform/internal/hostguard" |
| 26 | "github.com/carlosframework/platform/internal/hostops" |
| 27 | "github.com/carlosframework/platform/internal/hostwrite" |
| 28 | "github.com/carlosframework/platform/internal/moves" |
| 29 | "github.com/carlosframework/platform/internal/moves/archive" |
| 30 | "github.com/carlosframework/platform/internal/park" |
| 31 | "github.com/carlosframework/platform/internal/provision" |
| 32 | "github.com/carlosframework/platform/internal/registry" |
| 33 | "github.com/carlosframework/platform/internal/release" |
| 34 | ) |
| 35 | |
| 36 | // The moves owners for a landing probe (moves Phase 1 Plan 3 Task 5c; claim |
| 37 | // Plan 9 Group D, D5; rulings H2, H4, H13, H14, H16, H18). The claim's |
| 38 | // dispatcher (ClaimOwnerDispatch) hands this owner set only the move kinds, |
| 39 | // and it dispatches them again by the observed op's kind (ruling Q7): the |
| 40 | // restoring and reverting kinds go to the recovery owners |
| 41 | // (movesRecoveryOwners, movesrecoveryowners.go; Plan 4 Task 9), the moving |
| 42 | // kind is refused, and the landing is served here. Of the landing it serves |
| 43 | // one shape, a landing probe: the claim at landing/probing with this box |
| 44 | // executing as the move's target, the policy a probe, the registry row held |
| 45 | // by the move (and the committed wake, Task 7's, below). Everything else is |
| 46 | // refused with hostbox.ErrUnsupported or hostbox.ErrChanged, before any |
| 47 | // lease, stop or launch. |
| 48 | // |
| 49 | // The claim owns the admission, the restore, the launch, readiness, |
| 50 | // replication and the publication sequence (claimwakeassembly.go). This |
| 51 | // owner set supplies what moves knows: |
| 52 | // |
| 53 | // - Select (read-only, before admission): the landing is still |
| 54 | // admissible (its deadline, the record attempt within what T11 |
| 55 | // admitted, the install the record vouches for); the launch-input pin |
| 56 | // at spawn (§7.7, round-2 finding 14): the live release and the report |
| 57 | // of the selected launch's own environment are the pinned ones, else |
| 58 | // ProbeInputsChanged |
| 59 | // with the recorded sentence "configuration changed during the move", |
| 60 | // which the landing turns into its rollback (Task 6); then the |
| 61 | // admission per §7.14. The first probe is a cold admission into the |
| 62 | // prepared generation, the move id, after PreparedCheck (.files-gen and |
| 63 | // <dbPath>.gen name it and the holder's incarnation, and the journalled |
| 64 | // install manifest is the seal's N); its baseline is the build's pin |
| 65 | // (genselect.SelectPrepared, never an earlier stay's marker, H14); it |
| 66 | // keeps the flip's stamp and claims sealed coverage when the seal was |
| 67 | // built from the stopped database. A restarted attempt, once the move |
| 68 | // id is admitted, is a cold admission into a fresh generation: no kept |
| 69 | // stamp, no coverage, its baseline the prepared pin while the move id |
| 70 | // has published nothing and the certified remote (genselect.Select) |
| 71 | // once it has (H14). The launch identity is the pinned config and |
| 72 | // release (H4), CARLOS_MOVE_PROBE=1 is in the child's environment (H18) |
| 73 | // and the readiness path is the app's moves.ready_path (H12). |
| 74 | // - AdmissionEvidence: the same decisions and the same admissibility, |
| 75 | // re-derived and compared, pin included, against a baseline selected |
| 76 | // again, immediately before the CAS. |
| 77 | // - Materialize, before every start, under the admitted spawn selection, |
| 78 | // in the order ruled for C1 (fix round 1, finding 8): the held render |
| 79 | // and the held release, each under its own claim selection, held-config |
| 80 | // and held-release, through the existing held paths |
| 81 | // (appconfig.RenderHeldUnder, release.MaterializeHeldUnder) and |
| 82 | // MovesClaimPolicy's probing rows (H13), the render's etags compared |
| 83 | // with the pins before anything is selected or written; then the |
| 84 | // verified prepared marker, published under the admitted writer's |
| 85 | // fence; then the §5.1 start check (moves.StartCheck, not weakened) |
| 86 | // against the claim after the admission's CAS; only then the spawn. |
| 87 | // - Publish: Group D's call before replication finds the prepared marker |
| 88 | // Materialize published and records it; it writes only under the fence |
| 89 | // Materialize recorded for the session. The marker names the move id |
| 90 | // with the build's pin and install.json's digest, superseding the |
| 91 | // current marker (an earlier stay's, which Plan 2 keeps) by its exact |
| 92 | // ETag, or created when there is none. PublishFenced: a restarted |
| 93 | // attempt's fresh generation, after its own first root, as serving |
| 94 | // publishes it. PublicationEvidence: this process's receipt for the |
| 95 | // admitted session, read back. |
| 96 | // |
| 97 | // Production: the moves wake source (movesWakeSources, Plan 5 Task 9b) |
| 98 | // opens this owner set for the claim with both switches on, and its |
| 99 | // registration opens the gate that arms the parks. |
| 100 | type movesProbeOwners struct { |
| 101 | HomeRegion, FleetHomeRegion string |
| 102 | // Claims is the claim store (the deployment store in production). |
| 103 | Claims release.Store |
| 104 | Records MovesRecordReader |
| 105 | // ReadyPath is the app manifest's moves.ready_path, or "/". |
| 106 | ReadyPath func(ctx context.Context, account, app string) (string, error) |
| 107 | Data genselect.Reader |
| 108 | Bucket, Region, Box string |
| 109 | // DataDir is the root of the op journals (moves.JournalDir). |
| 110 | DataDir string |
| 111 | FilesDir func(host string) (dir, envPath string) |
| 112 | Markers ClaimMarkerStore |
| 113 | Route func(host string) (*registry.Route, error) |
| 114 | // Launch is the held row's launch: the live executable, its digest, |
| 115 | // arguments, environment and version (the claim's launch snapshot), and |
| 116 | // the moves env report of that same environment, computed at capture |
| 117 | // from the one composition that produced the env (claimMoveLaunchSource: |
| 118 | // ExecBackend.LaunchLayers, moves.ReportLaunch). The report's Digest is |
| 119 | // the launch's move-portable pin. It is the moves owners' own value: |
| 120 | // never persisted, and never digested by the claim (fix round 4). |
| 121 | Launch func(context.Context, registry.Route) (ClaimServingLaunch, moves.EnvReport, error) |
| 122 | // EnvKey is the env report's HMAC key (moves.ReportKeyFromSecrets), the |
| 123 | // key Launch's report is made with. The pin is checked against the |
| 124 | // report of the selected launch's own environment, so no report can be |
| 125 | // decoupled from what runs (fix round 1, finding 3). Normalisation |
| 126 | // contract: the report covers the launch's K=V list, last value wins |
| 127 | // (childEnv's dedupe), with the probe-only variables (probeOnlyVars: |
| 128 | // CARLOS_MOVE_PROBE) removed, since the source never ran with them and |
| 129 | // the probe adds CARLOS_MOVE_PROBE=1 after the pin, and its digest is the |
| 130 | // move-portable one (moves.PortableDigest). The production Launch is this |
| 131 | // box's own composition (claim Plan 10's launch accessor). |
| 132 | EnvKey []byte |
| 133 | // Admitted is the number of probe attempts T11 admitted (H10). |
| 134 | Admitted func(context.Context, *provision.Record) (int, error) |
| 135 | // Works selects a held action's claim work at the admitted capture |
| 136 | // (hostbox.WorkSource.SelectAt). |
| 137 | Works func(ctx context.Context, host, action string, captured hostguard.Captured) (hostwrite.Work, error) |
| 138 | Render func(ctx context.Context, rt registry.Route, op string, work func(appconfig.HeldRender) (hostwrite.Work, error)) (appconfig.HeldRender, error) |
| 139 | // Release makes version live for the held row before a start. On a box |
| 140 | // whose agent cannot write /opt/carlos it is heldReleaseProbeRelease |
| 141 | // (Task 7b): root publishes, inside root's own held-release selection. |
| 142 | Release func(ctx context.Context, rt registry.Route, version string, work hostwrite.Work) error |
| 143 | NewID func() string |
| 144 | Now func() time.Time |
| 145 | // Page pages an operator for a refused start check, once per op |
| 146 | // (movespaging.go, §13): PageOperator once claim Plan 10 registers this |
| 147 | // owner set; nil, nothing pages. |
| 148 | Page func(ctx context.Context, level, subject, body string) error |
| 149 | // Files is the moves C8 files manifest provider (claim §6.2a P6-D3; |
| 150 | // Task 6 builds it). The recovery owners' Materialize establishes the |
| 151 | // admitted generation's files manifest through it before every start; |
| 152 | // without it every recovery kind is refused, typed, before anything is |
| 153 | // read (movesrecoveryowners.go). |
| 154 | Files ClaimFilesManifest |
| 155 | // pageWarned is the pages warned unsent for want of a pager. |
| 156 | pageWarned sync.Map |
| 157 | |
| 158 | mu sync.Mutex |
| 159 | published map[string]ClaimPublicationReceipt |
| 160 | // envPins are the move-portable pins Select captured, by host, each |
| 161 | // bound to the exact launch environment it was computed for |
| 162 | // (captureEnvPin): what Materialize checks the same inputs against. |
| 163 | // Memory only, never persisted, never digested by the claim. Each lives |
| 164 | // as long as its wake (holdEnvPin) and never past its deadline. |
| 165 | envPins map[string]capturedEnvPin |
| 166 | envPinSeq uint64 |
| 167 | // fences are the admitted spawn selections Materialize ran under, by |
| 168 | // host and session: the live fence for the session's marker writes. |
| 169 | fences map[string]hostwrite.Work |
| 170 | // lossValidated is each host's last LossSources answer, which the |
| 171 | // recovery's AdmissionEvidence of the same admission consumes |
| 172 | // (movesloss.go). |
| 173 | lossValidated map[string]lossValidation |
| 174 | } |
| 175 | |
| 176 | var ( |
| 177 | _ ClaimWakeInputs = (*movesProbeOwners)(nil) |
| 178 | _ ClaimFencedPublication = (*movesProbeOwners)(nil) |
| 179 | ) |
| 180 | |
| 181 | // ProbeInputsChanged is a probe refused because a launch input is not the |
| 182 | // one pinned at readiness. Reason is the landing's recorded rollback |
| 183 | // sentence (provision.RollbackConfigChanged); the landing records it. |
| 184 | type ProbeInputsChanged struct{ Reason string } |
| 185 | |
| 186 | func (e *ProbeInputsChanged) Error() string { |
| 187 | return "host: the probe's launch inputs are not the pinned ones: " + e.Reason |
| 188 | } |
| 189 | |
| 190 | func probeInputsChanged() error { |
| 191 | return &ProbeInputsChanged{Reason: provision.RollbackConfigChanged} |
| 192 | } |
| 193 | |
| 194 | // owners is the owner set's ClaimKindOwners. |
| 195 | func (m *movesProbeOwners) owners() ClaimKindOwners { |
| 196 | return ClaimKindOwners{Inputs: m, AdmissionEvidence: m.AdmissionEvidence, PublicationEvidence: m.PublicationEvidence} |
| 197 | } |
| 198 | |
| 199 | // MovesOwners is ClaimMovesOwners for the probe: this owner set and ops, |
| 200 | // with the moves loss owner (the resolve owner's signed-intent policy, |
| 201 | // hostops.MoveLossRecoveryOwner) and the loss baseline validator |
| 202 | // (LossSources, movesloss.go) the claim's writer admits a declared or fenced |
| 203 | // return through. ProbeReady stays nil, so the claim binds its own probe |
| 204 | // readiness check (claimProbeSocketReady: the declared path, 2xx only, a |
| 205 | // peer in the started role's cgroup; H12). |
| 206 | func (m *movesProbeOwners) MovesOwners(ops OperationAttempts) ClaimMovesOwners { |
| 207 | return ClaimMovesOwners{LossRecovery: hostops.MoveLossRecoveryOwner(), LossSources: m.LossSources, ClaimKindOwners: m.owners(), Operations: ops} |
| 208 | } |
| 209 | |
| 210 | func (m *movesProbeOwners) valid() bool { |
| 211 | return m != nil && m.Claims != nil && m.Records != nil && m.ReadyPath != nil && m.Data != nil && m.Bucket != "" && m.Region != "" && m.Box != "" && |
| 212 | m.DataDir != "" && m.FilesDir != nil && m.Markers != nil && m.Route != nil && m.Launch != nil && len(m.EnvKey) != 0 && m.Admitted != nil && m.Works != nil && |
| 213 | m.Render != nil && m.Release != nil && m.NewID != nil && m.Now != nil |
| 214 | } |
| 215 | |
| 216 | // probeLanding is what one landing probe's checks read. |
| 217 | type probeLanding struct { |
| 218 | homeRegion string |
| 219 | // committed is a landing wake after the commit (Plan 3 Task 7): the |
| 220 | // claim at landing/committed, the record committed. |
| 221 | committed bool |
| 222 | op string |
| 223 | holder *hostclaim.Holder |
| 224 | rec *provision.Record |
| 225 | seal moves.Seal |
| 226 | manifest moves.InstallManifest |
| 227 | } |
| 228 | |
| 229 | // landing checks that o is a landing probe this box may run and reads the |
| 230 | // record, the seal and the journalled install manifest. |
| 231 | func (m *movesProbeOwners) landing(ctx context.Context, o hostclaim.Observed) (probeLanding, error) { |
| 232 | var v probeLanding |
| 233 | if !m.valid() { |
| 234 | return v, hostwrite.ErrUnavailable |
| 235 | } |
| 236 | if o.Absent || o.Claim == nil || o.Claim.Holder == nil || o.Claim.Op.Kind != hostclaim.KindLanding { |
| 237 | return v, hostbox.ErrUnsupported |
| 238 | } |
| 239 | c := o.Claim |
| 240 | var bindingErr error |
| 241 | v.homeRegion, bindingErr = park.VerifiedHome(m.FleetHomeRegion, m.HomeRegion, m.Region, c.Op.Intent["home_region"]) |
| 242 | if bindingErr != nil { |
| 243 | return v, bindingErr |
| 244 | } |
| 245 | switch c.Op.Phase { |
| 246 | case hostclaim.PhaseLandingProbing: |
| 247 | case hostclaim.PhaseLandingCommitted: |
| 248 | v.committed = true |
| 249 | default: |
| 250 | return v, hostbox.ErrChanged |
| 251 | } |
| 252 | if c.Host != o.Host || c.Op.ID == "" || c.Op.Exec != m.Box || c.Op.Intent["target"] != m.Box || |
| 253 | c.Op.Intent["source"] == m.Box || !park.SameRegion(m.HomeRegion, c.Op.Intent["to"], m.Region) || len(c.Op.Participants) != 2 || c.Op.Participants[1] != m.Box { |
| 254 | return v, hostbox.ErrChanged |
| 255 | } |
| 256 | for _, lost := range c.Lost { |
| 257 | if lost == m.Box { |
| 258 | return v, hostbox.ErrChanged |
| 259 | } |
| 260 | } |
| 261 | v.op, v.holder = c.Op.ID, c.Holder |
| 262 | rec, err := m.Records(ctx, v.holder.Account, v.holder.App, o.Host) |
| 263 | if err != nil { |
| 264 | return v, err |
| 265 | } |
| 266 | if rec == nil || rec.Host != o.Host || rec.Move == nil { |
| 267 | return v, hostbox.ErrChanged |
| 268 | } |
| 269 | mv := rec.Move |
| 270 | if park.PlacementRegion(mv.From) == "" || mv.ID != v.op || mv.Incarnation != v.holder.Incarnation || !park.PlacementMatches(m.HomeRegion, mv.To, m.Region) || mv.Inputs == nil || mv.Inputs.Release == "" { |
| 271 | return v, hostbox.ErrChanged |
| 272 | } |
| 273 | if v.committed { |
| 274 | if mv.State != provision.MoveCommitted { |
| 275 | return v, hostbox.ErrChanged |
| 276 | } |
| 277 | } else if mv.State != provision.MoveLanding || mv.LandingPhase != provision.LandProbing || mv.Reason != "" { |
| 278 | return v, hostbox.ErrChanged |
| 279 | } |
| 280 | v.rec = rec |
| 281 | if v.seal, err = journalledSeal(c); err != nil { |
| 282 | return v, err |
| 283 | } |
| 284 | j, err := moves.ReadJournal(m.DataDir, o.Host, v.op) |
| 285 | if err != nil { |
| 286 | return v, err |
| 287 | } |
| 288 | raw, ok := j.Steps[landStepInstall] |
| 289 | if !ok { |
| 290 | return v, fmt.Errorf("%w: the landing journalled no install manifest", hostbox.ErrChanged) |
| 291 | } |
| 292 | if err := json.Unmarshal([]byte(raw), &v.manifest); err != nil { |
| 293 | return v, fmt.Errorf("the journalled install manifest: %w", err) |
| 294 | } |
| 295 | if v.manifest.Op != v.op || v.manifest.Host != o.Host || v.manifest.N != v.seal.N || v.manifest.N == 0 || v.manifest.SnapshotKey == "" || v.manifest.SnapshotVersion == "" { |
| 296 | return v, fmt.Errorf("%w: the install manifest is not this move's build at the seal", hostbox.ErrChanged) |
| 297 | } |
| 298 | return v, nil |
| 299 | } |
| 300 | |
| 301 | // heldRoute is the registry row the probe runs: this host, held by the move. |
| 302 | func heldRoute(rt registry.Route, host, op string) error { |
| 303 | if rt.Host != host || !registry.IsHold(rt.Status) || rt.HoldOp != op || !filepath.IsAbs(rt.DBPath) { |
| 304 | return hostbox.ErrChanged |
| 305 | } |
| 306 | return nil |
| 307 | } |
| 308 | |
| 309 | // firstAdmission reports whether the holder's writer is still the move's |
| 310 | // source: the flip leaves the source's session in place, and every |
| 311 | // admission on this box makes it the writer. A first admission is the |
| 312 | // prepared one; any later one is a restarted attempt into a fresh |
| 313 | // generation. It reads no admission history, so trimming cannot change it. |
| 314 | func firstAdmission(h *hostclaim.Holder, box string) bool { |
| 315 | return h.WriterBox != box |
| 316 | } |
| 317 | |
| 318 | // admissible is the landing's own word that an admission may happen now |
| 319 | // (fix round 1, finding 1): before the landing's deadline (the earlier of |
| 320 | // cut_deadline and landing_deadline, H8), at a record attempt within what |
| 321 | // T11 admitted (H10), over the install the record vouches for. |
| 322 | func (m *movesProbeOwners) admissible(ctx context.Context, v probeLanding) error { |
| 323 | mv := v.rec.Move |
| 324 | d, _ := landingDeadlineOf(mv) |
| 325 | if d == nil || !m.Now().Before(*d) { |
| 326 | return fmt.Errorf("%w: the landing is past its deadline", hostbox.ErrChanged) |
| 327 | } |
| 328 | admitted, err := m.Admitted(ctx, v.rec) |
| 329 | if err != nil { |
| 330 | return err |
| 331 | } |
| 332 | if mv.Landing == nil || mv.Landing.Attempts < 1 || mv.Landing.Attempts > moves.AttemptsFor(moves.ProbeLanding, admitted) { |
| 333 | return fmt.Errorf("%w: the record's probe attempt is not one T11 admitted", hostbox.ErrChanged) |
| 334 | } |
| 335 | if mv.Landing.InstalledDBSHA256 != "sha256:"+v.manifest.DBSHA256 { |
| 336 | return fmt.Errorf("%w: the record does not vouch for this install", hostbox.ErrChanged) |
| 337 | } |
| 338 | return nil |
| 339 | } |
| 340 | |
| 341 | // probeOnlyVars are the variables the probe adds to the launch and the |
| 342 | // readiness pin never saw. |
| 343 | var probeOnlyVars = map[string]bool{"CARLOS_MOVE_PROBE": true} |
| 344 | |
| 345 | // capturedEnvPin is a launch environment's move-portable pin as Select |
| 346 | // computed it at capture (Launch's report), bound to that environment by |
| 347 | // its fingerprint. seq names this capture, so the end of an earlier wake |
| 348 | // never drops a later one; until is the wake's deadline, after which the |
| 349 | // entry is gone whether or not the wake ever reached Materialize; held is |
| 350 | // set once its wake's start loop holds it (holdEnvPin). |
| 351 | type capturedEnvPin struct { |
| 352 | env, pin string |
| 353 | seq uint64 |
| 354 | until time.Time |
| 355 | held bool |
| 356 | } |
| 357 | |
| 358 | // envPinHold bounds a captured pin whose wake names no deadline (the |
| 359 | // committed wake's policy). |
| 360 | const envPinHold = 15 * time.Minute |
| 361 | |
| 362 | // envFingerprint identifies a launch environment exactly: every entry, in |
| 363 | // order. |
| 364 | func envFingerprint(env []string) string { |
| 365 | sum := sha256.Sum256([]byte(strings.Join(env, "\x00"))) |
| 366 | return hex.EncodeToString(sum[:]) |
| 367 | } |
| 368 | |
| 369 | // captureEnvPin records the pin Select computed for the inputs it returns |
| 370 | // (env is ClaimLaunchInputs.Env), replacing any earlier one for host. The |
| 371 | // entry lasts until its wake ends (holdEnvPin) and never past deadline (the |
| 372 | // wake's StartPolicy.Deadline, or envPinHold when it names none): a wake |
| 373 | // abandoned before Materialize leaves nothing behind longer than that. |
| 374 | // Every capture first drops the expired entries. |
| 375 | func (m *movesProbeOwners) captureEnvPin(host string, env []string, pin string, deadline time.Time) { |
| 376 | now := m.now() |
| 377 | if deadline.IsZero() { |
| 378 | deadline = now.Add(envPinHold) |
| 379 | } |
| 380 | m.mu.Lock() |
| 381 | defer m.mu.Unlock() |
| 382 | if m.envPins == nil { |
| 383 | m.envPins = map[string]capturedEnvPin{} |
| 384 | } |
| 385 | m.pruneEnvPins(now) |
| 386 | m.envPinSeq++ |
| 387 | m.envPins[host] = capturedEnvPin{env: envFingerprint(env), pin: pin, seq: m.envPinSeq, until: deadline} |
| 388 | } |
| 389 | |
| 390 | // now is the owners' clock, the wall clock when none is set. |
| 391 | func (m *movesProbeOwners) now() time.Time { |
| 392 | if m.Now == nil { |
| 393 | return time.Now() |
| 394 | } |
| 395 | return m.Now() |
| 396 | } |
| 397 | |
| 398 | // pruneEnvPins drops the entries past their deadline. m.mu is held. |
| 399 | func (m *movesProbeOwners) pruneEnvPins(now time.Time) { |
| 400 | for host, c := range m.envPins { |
| 401 | if !now.Before(c.until) { |
| 402 | delete(m.envPins, host) |
| 403 | } |
| 404 | } |
| 405 | } |
| 406 | |
| 407 | // capturedPin is the pin Select captured for exactly env on host; false |
| 408 | // when Select captured none for it in this process, or it has expired |
| 409 | // (Materialize then refuses, fail closed). |
| 410 | func (m *movesProbeOwners) capturedPin(host string, env []string) (string, bool) { |
| 411 | m.mu.Lock() |
| 412 | defer m.mu.Unlock() |
| 413 | m.pruneEnvPins(m.now()) |
| 414 | c, ok := m.envPins[host] |
| 415 | if !ok || c.env != envFingerprint(env) { |
| 416 | return "", false |
| 417 | } |
| 418 | return c.pin, true |
| 419 | } |
| 420 | |
| 421 | // holdEnvPin ties host's captured pin to its wake: ctx is the context the |
| 422 | // activator's start loop passes Materialize, which it cancels when the |
| 423 | // wake ends, completed, failed or abandoned. Every retry of the same |
| 424 | // launch runs under the same context and finds the entry; the end of the |
| 425 | // wake drops it, and never a later capture for the same host (seq). Held |
| 426 | // once per capture. |
| 427 | func (m *movesProbeOwners) holdEnvPin(ctx context.Context, host string) { |
| 428 | m.mu.Lock() |
| 429 | c, ok := m.envPins[host] |
| 430 | if !ok || c.held { |
| 431 | m.mu.Unlock() |
| 432 | return |
| 433 | } |
| 434 | c.held = true |
| 435 | m.envPins[host] = c |
| 436 | m.mu.Unlock() |
| 437 | seq := c.seq |
| 438 | context.AfterFunc(ctx, func() { m.dropEnvPin(host, seq) }) |
| 439 | } |
| 440 | |
| 441 | // dropEnvPin removes host's captured pin if it is still capture seq. |
| 442 | func (m *movesProbeOwners) dropEnvPin(host string, seq uint64) { |
| 443 | m.mu.Lock() |
| 444 | defer m.mu.Unlock() |
| 445 | if c, ok := m.envPins[host]; ok && c.seq == seq { |
| 446 | delete(m.envPins, host) |
| 447 | } |
| 448 | } |
| 449 | |
| 450 | // localEnvDigest is the digest of the whole launch environment, every |
| 451 | // variable but the probe-only ones (moves.LocalDigest): the recovery's |
| 452 | // check against inputs this same box recorded just before the wake (fix |
| 453 | // round 2, platform-1e's same-box rule). |
| 454 | func localEnvDigest(key []byte, env []string) string { |
| 455 | return moves.LocalDigest(key, launchVars(env)) |
| 456 | } |
| 457 | |
| 458 | // launchVars is a launch's K=V list as a map, last value wins, the |
| 459 | // probe-only variables dropped. |
| 460 | func launchVars(env []string) map[string]string { |
| 461 | vars := map[string]string{} |
| 462 | for _, kv := range env { |
| 463 | k, v, ok := strings.Cut(kv, "=") |
| 464 | if !ok || probeOnlyVars[k] { |
| 465 | continue |
| 466 | } |
| 467 | vars[k] = v |
| 468 | } |
| 469 | return vars |
| 470 | } |
| 471 | |
| 472 | // manifestDigest is install.json's digest: the build writes the manifest |
| 473 | // as json.Marshal of the journalled value (moves.BuildGeneration). |
| 474 | func manifestDigest(m moves.InstallManifest) (string, error) { |
| 475 | raw, err := json.Marshal(m) |
| 476 | if err != nil { |
| 477 | return "", err |
| 478 | } |
| 479 | sum := sha256.Sum256(raw) |
| 480 | return hex.EncodeToString(sum[:]), nil |
| 481 | } |
| 482 | |
| 483 | func manifestPin(m moves.InstallManifest) genselect.Pin { |
| 484 | return genselect.Pin{SnapshotKey: m.SnapshotKey, SnapshotVersion: m.SnapshotVersion, SnapshotTXID: m.N, SnapshotCRC64: m.SnapshotCRC64} |
| 485 | } |
| 486 | |
| 487 | // landingMarker reports whether the location's current marker already |
| 488 | // names this landing: the move id's verified prepared marker, or a fresh |
| 489 | // generation proved to descend from the move's admission (descendsFromMove). |
| 490 | // Anything else (an earlier stay's, though its cold admission on this box |
| 491 | // and region is still in the history; another incarnation's) is superseded |
| 492 | // by the prepared one, never taken for a restart (fix round 2, finding 8). |
| 493 | func (m *movesProbeOwners) landingMarker(marker *genselect.Marker, host string, v probeLanding, digest string) bool { |
| 494 | h := v.holder |
| 495 | if marker == nil || marker.Incarnation != h.Incarnation || marker.Host != host || marker.Box != m.Box { |
| 496 | return false |
| 497 | } |
| 498 | if marker.Op == v.op { |
| 499 | return marker.Pin == manifestPin(v.manifest) && marker.ManifestSHA256 == digest |
| 500 | } |
| 501 | return m.descendsFromMove(h, marker.Op, v.op) |
| 502 | } |
| 503 | |
| 504 | // descendsFromMove is positive proof that generation g is a restart of this |
| 505 | // landing: walking back from g, each step a cold admission on this box and |
| 506 | // region into the current generation, its predecessor chain reaches the |
| 507 | // move's prepared generation (the admission into the move id itself, which |
| 508 | // names the op). A chain that leaves this box or region, loops, or is |
| 509 | // trimmed before the move id is no proof. |
| 510 | func (m *movesProbeOwners) descendsFromMove(h *hostclaim.Holder, g, op string) bool { |
| 511 | seen := map[string]bool{} |
| 512 | for g != "" && g != op && !seen[g] { |
| 513 | seen[g] = true |
| 514 | found := false |
| 515 | for i := len(h.Admissions) - 1; i >= 0; i-- { |
| 516 | a := h.Admissions[i] |
| 517 | if a.Generation != g || a.Mode != "cold" { |
| 518 | continue |
| 519 | } |
| 520 | if a.Box != m.Box || !park.SameRegion(m.HomeRegion, a.Region, m.Region) { |
| 521 | return false |
| 522 | } |
| 523 | g, found = a.Predecessor, true |
| 524 | break |
| 525 | } |
| 526 | if !found { |
| 527 | return false |
| 528 | } |
| 529 | } |
| 530 | if g != op { |
| 531 | return false |
| 532 | } |
| 533 | for _, a := range h.Admissions { |
| 534 | if a.Generation == op && a.Box == m.Box && park.SameRegion(m.HomeRegion, a.Region, m.Region) && a.Mode == "cold" { |
| 535 | return true |
| 536 | } |
| 537 | } |
| 538 | return false |
| 539 | } |
| 540 | |
| 541 | // predecessorRegion is AdmitWriter's predecessor region: the last |
| 542 | // admission's, or the move's source region before any. |
| 543 | func predecessorRegion(h *hostclaim.Holder, from string) (string, error) { |
| 544 | if !provision.ValidRegion(from) { |
| 545 | return "", hostbox.ErrChanged |
| 546 | } |
| 547 | if n := len(h.Admissions); n > 0 { |
| 548 | return h.Admissions[n-1].Region, nil |
| 549 | } |
| 550 | return park.PlacementRegion(from), nil |
| 551 | } |
| 552 | |
| 553 | func (m *movesProbeOwners) request(host string, h *hostclaim.Holder) genselect.Request { |
| 554 | return genselect.Request{Bucket: m.Bucket, Region: m.Region, Host: host, Holder: park.HolderRegionView(*h, m.HomeRegion, m.Region)} |
| 555 | } |
| 556 | |
| 557 | func preparedPin(v probeLanding) genselect.PreparedPin { |
| 558 | // The build's own pin, from the journalled manifest. |
| 559 | return genselect.PreparedPin{Generation: v.op, SnapshotKey: v.manifest.SnapshotKey, SnapshotVersion: v.manifest.SnapshotVersion, SnapshotTXID: v.manifest.N, SnapshotCRC64: v.manifest.SnapshotCRC64} |
| 560 | } |
| 561 | |
| 562 | func markerKey(h *hostclaim.Holder) string { |
| 563 | return strings.TrimSuffix(h.Location, "/") + "/generation.json" |
| 564 | } |
| 565 | |
| 566 | // currentMarker is the location's current generation.json version, or nil. |
| 567 | func (m *movesProbeOwners) currentMarker(ctx context.Context, h *hostclaim.Holder) (*claimMarkerVersion, error) { |
| 568 | return currentMarker(ctx, m.Data, m.Bucket, h) |
| 569 | } |
| 570 | |
| 571 | // currentMarker is the location's current generation.json version, or |
| 572 | // nil when the key has none; a delete marker, or two latest versions, is |
| 573 | // ErrChanged. The moves probe owners and region HA's relocation inputs |
| 574 | // share it (8d fix round 1, M3). |
| 575 | func currentMarker(ctx context.Context, data genselect.Reader, bucket string, h *hostclaim.Holder) (*claimMarkerVersion, error) { |
| 576 | key := markerKey(h) |
| 577 | versions, err := data.ListVersions(ctx, bucket, key) |
| 578 | if err != nil { |
| 579 | return nil, err |
| 580 | } |
| 581 | var current *claimMarkerVersion |
| 582 | for _, v := range versions { |
| 583 | if v.Key == key && v.IsLatest { |
| 584 | if current != nil || v.DeleteMarker { |
| 585 | return nil, hostbox.ErrChanged |
| 586 | } |
| 587 | current = &claimMarkerVersion{ID: v.VersionID, ETag: v.ETag} |
| 588 | } |
| 589 | } |
| 590 | return current, nil |
| 591 | } |
| 592 | |
| 593 | // baseline is the admission's baseline (H14): the build's pin for the |
| 594 | // prepared generation; for a restarted attempt, the prepared pin while the |
| 595 | // location's marker is not this landing's (the move id published nothing |
| 596 | // beyond the build; an earlier stay's pin is never a baseline), and the |
| 597 | // certified remote once it is. |
| 598 | func (m *movesProbeOwners) baseline(ctx context.Context, host string, v probeLanding, prepared bool) (*genselect.Plan, error) { |
| 599 | if prepared { |
| 600 | return genselect.SelectPrepared(ctx, m.Data, m.request(host, v.holder), preparedPin(v)) |
| 601 | } |
| 602 | digest, err := manifestDigest(v.manifest) |
| 603 | if err != nil { |
| 604 | return nil, err |
| 605 | } |
| 606 | marker, err := m.readMarker(ctx, v.holder) |
| 607 | if err != nil { |
| 608 | return nil, err |
| 609 | } |
| 610 | if !m.landingMarker(marker, host, v, digest) { |
| 611 | return genselect.SelectPrepared(ctx, m.Data, m.request(host, v.holder), preparedPin(v)) |
| 612 | } |
| 613 | return genselect.Select(ctx, m.Data, m.request(host, v.holder)) |
| 614 | } |
| 615 | |
| 616 | // localMarkers reads .files-gen (nil when absent) and <dbPath>.gen ("" when |
| 617 | // absent). |
| 618 | func (m *movesProbeOwners) localMarkers(host, dbPath string) (*moves.FilesGen, string, error) { |
| 619 | dir, _ := m.FilesDir(host) |
| 620 | if !filepath.IsAbs(dir) { |
| 621 | return nil, "", hostwrite.ErrUnavailable |
| 622 | } |
| 623 | var files *moves.FilesGen |
| 624 | raw, err := os.ReadFile(filepath.Join(dir, archive.MarkerName)) |
| 625 | switch { |
| 626 | case errors.Is(err, fs.ErrNotExist): |
| 627 | case err != nil: |
| 628 | return nil, "", err |
| 629 | default: |
| 630 | var fg moves.FilesGen |
| 631 | dec := json.NewDecoder(bytes.NewReader(raw)) |
| 632 | dec.DisallowUnknownFields() |
| 633 | if err := dec.Decode(&fg); err != nil { |
| 634 | return nil, "", fmt.Errorf("%w: %v", moves.ErrStartFiles, err) |
| 635 | } |
| 636 | files = &fg |
| 637 | } |
| 638 | db, err := hostbox.ReadGenerationMarker(dbPath + ".gen") |
| 639 | if err != nil { |
| 640 | return nil, "", err |
| 641 | } |
| 642 | return files, db, nil |
| 643 | } |
| 644 | |
| 645 | // launchIdentity is the probe's spawn identity (H4): the pinned config and |
| 646 | // release, bound to the record by MovesClaimPolicy. |
| 647 | func launchIdentity(in *provision.MoveInputs) hostwrite.Identity { |
| 648 | return hostwrite.Identity{Config: HeldConfigIdentity(in.BundleETags, in.OverlayETag).Config, Release: in.Release} |
| 649 | } |
| 650 | |
| 651 | // probeEnv is the launch environment with CARLOS_MOVE_PROBE=1 (H18), the |
| 652 | // one entry for that name. |
| 653 | func probeEnv(env []string) []string { |
| 654 | out := make([]string, 0, len(env)+1) |
| 655 | for _, kv := range env { |
| 656 | if !strings.HasPrefix(kv, "CARLOS_MOVE_PROBE=") { |
| 657 | out = append(out, kv) |
| 658 | } |
| 659 | } |
| 660 | return append(out, "CARLOS_MOVE_PROBE=1") |
| 661 | } |
| 662 | |
| 663 | // Select chooses the probe's launch inputs and admission. It is read-only. |
| 664 | func (m *movesProbeOwners) Select(ctx context.Context, rt registry.Route, o hostclaim.Observed, p activator.StartPolicy) (ClaimLaunchInputs, error) { |
| 665 | if recoveryKind(o) { |
| 666 | return m.recovery().Select(ctx, rt, o, p) |
| 667 | } |
| 668 | v, err := m.landing(ctx, o) |
| 669 | if err != nil { |
| 670 | return ClaimLaunchInputs{}, err |
| 671 | } |
| 672 | if v.committed { |
| 673 | if p.Probe || p.Prepared != nil || p.Inputs != nil { |
| 674 | return ClaimLaunchInputs{}, hostbox.ErrUnsupported |
| 675 | } |
| 676 | return m.selectCommitted(ctx, rt, o, v, p.Deadline) |
| 677 | } |
| 678 | if !p.Probe { |
| 679 | return ClaimLaunchInputs{}, hostbox.ErrUnsupported |
| 680 | } |
| 681 | pins := v.rec.Move.Inputs |
| 682 | if (p.Inputs != nil && !reflect.DeepEqual(*p.Inputs, *pins)) || (p.Prepared != nil && *p.Prepared != v.manifest) { |
| 683 | return ClaimLaunchInputs{}, hostbox.ErrChanged |
| 684 | } |
| 685 | if err := heldRoute(rt, o.Host, v.op); err != nil { |
| 686 | return ClaimLaunchInputs{}, err |
| 687 | } |
| 688 | if err := m.admissible(ctx, v); err != nil { |
| 689 | return ClaimLaunchInputs{}, err |
| 690 | } |
| 691 | h := v.holder |
| 692 | prepared := firstAdmission(h, m.Box) |
| 693 | predecessor, err := predecessorRegion(h, v.rec.Move.From) |
| 694 | if err != nil { |
| 695 | return ClaimLaunchInputs{}, err |
| 696 | } |
| 697 | admission := hostclaim.Admission{ObservedOpID: v.op, ObservedPhase: o.Claim.Op.Phase, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: m.Box, Mode: "cold", |
| 698 | KeepStamp: h.Stamp > h.Session, Region: m.Region, PredecessorRegion: predecessor} |
| 699 | if prepared { |
| 700 | files, db, err := m.localMarkers(o.Host, rt.DBPath) |
| 701 | if err != nil { |
| 702 | return ClaimLaunchInputs{}, err |
| 703 | } |
| 704 | if err := moves.PreparedCheck(v.op, *h, files, db); err != nil { |
| 705 | return ClaimLaunchInputs{}, err |
| 706 | } |
| 707 | admission.NewGeneration = v.op |
| 708 | if v.seal.FromDB { |
| 709 | admission.Coverage = "sealed" |
| 710 | } |
| 711 | } else { |
| 712 | admission.NewGeneration = m.NewID() |
| 713 | if admission.NewGeneration == v.op || !claimFreshGeneration(h, admission.NewGeneration) { |
| 714 | return ClaimLaunchInputs{}, hostwrite.ErrUnavailable |
| 715 | } |
| 716 | } |
| 717 | plan, err := m.baseline(ctx, o.Host, v, prepared) |
| 718 | if err != nil { |
| 719 | return ClaimLaunchInputs{}, err |
| 720 | } |
| 721 | admission.RemoteMaxTXID = plan.MaxTXID |
| 722 | ready, err := m.ReadyPath(ctx, h.Account, h.App) |
| 723 | if err != nil { |
| 724 | return ClaimLaunchInputs{}, err |
| 725 | } |
| 726 | // The launch-input pin at spawn: the authoritative check, after |
| 727 | // readiness's (Plan 2's configured), since the inputs can change between. |
| 728 | l, report, err := m.Launch(ctx, rt) |
| 729 | if err != nil { |
| 730 | return ClaimLaunchInputs{}, err |
| 731 | } |
| 732 | if !filepath.IsAbs(l.Executable) || !claimHexDigest(l.ExecutableSHA256) || l.Version == "" { |
| 733 | return ClaimLaunchInputs{}, hostwrite.ErrUnavailable |
| 734 | } |
| 735 | if l.Version != pins.Release { |
| 736 | return ClaimLaunchInputs{}, probeInputsChanged() |
| 737 | } |
| 738 | // A v1 pin is refused whatever it hashes: it cannot tell an instance's |
| 739 | // box variable from the box's (moves.EnvReportVersion). The launch's own |
| 740 | // pin is its report's, computed at capture with its env. |
| 741 | if !moves.CurrentEnvPin(pins.EnvReport) || report.Digest != pins.EnvReport { |
| 742 | return ClaimLaunchInputs{}, probeInputsChanged() |
| 743 | } |
| 744 | in := ClaimLaunchInputs{Roles: []string{"main"}, Identity: launchIdentity(pins), Baseline: plan, Admission: admission, |
| 745 | Executable: l.Executable, ExecutableSHA256: l.ExecutableSHA256, Args: append([]string(nil), l.Args...), Env: probeEnv(l.Env), |
| 746 | DBPath: rt.DBPath, Version: l.Version, Staged: l.Staged, Limit: l.Limit, Prepared: prepared, ReadyPath: ready} |
| 747 | m.captureEnvPin(rt.Host, in.Env, report.Digest, p.Deadline) |
| 748 | return in, nil |
| 749 | } |
| 750 | |
| 751 | // AdmissionEvidence re-derives Select's admission against the observation |
| 752 | // and a baseline selected again, immediately before the admission's CAS. |
| 753 | func (m *movesProbeOwners) AdmissionEvidence(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, baseline *genselect.Plan) error { |
| 754 | if recoveryKind(o) { |
| 755 | return m.recovery().AdmissionEvidence(ctx, o, in, baseline) |
| 756 | } |
| 757 | v, err := m.landing(ctx, o) |
| 758 | if err != nil { |
| 759 | return err |
| 760 | } |
| 761 | if v.committed { |
| 762 | return m.committedEvidence(ctx, o, v, in, baseline) |
| 763 | } |
| 764 | if err := m.admissible(ctx, v); err != nil { |
| 765 | return err |
| 766 | } |
| 767 | h := v.holder |
| 768 | predecessor, err := predecessorRegion(h, v.rec.Move.From) |
| 769 | if err != nil { |
| 770 | return err |
| 771 | } |
| 772 | if in.ObservedOpID != v.op || in.ObservedPhase != o.Claim.Op.Phase || in.ObservedSession != h.Session || in.ObservedGeneration != h.Generation || in.Box != m.Box || !park.SameRegion(m.HomeRegion, in.Region, m.Region) || |
| 773 | in.Mode != "cold" || in.PredecessorCertified || in.LossID != "" || len(in.AdditionalWritesLost) != 0 || in.KeepStamp != (h.Stamp > h.Session) || |
| 774 | !park.SameRegion(v.homeRegion, in.PredecessorRegion, predecessor) || baseline == nil || in.RemoteMaxTXID != baseline.MaxTXID { |
| 775 | return hostbox.ErrChanged |
| 776 | } |
| 777 | prepared := firstAdmission(h, m.Box) |
| 778 | switch { |
| 779 | case prepared: |
| 780 | coverage := "" |
| 781 | if v.seal.FromDB { |
| 782 | coverage = "sealed" |
| 783 | } |
| 784 | pin := manifestPin(v.manifest) |
| 785 | if in.NewGeneration != v.op || in.Coverage != coverage || baseline.Generation != v.op || baseline.Pin == nil || *baseline.Pin != pin || |
| 786 | baseline.MaxTXID != v.manifest.N || v.manifest.N != v.seal.N { |
| 787 | return hostbox.ErrChanged |
| 788 | } |
| 789 | rt, err := m.route(o.Host, v.op) |
| 790 | if err != nil { |
| 791 | return err |
| 792 | } |
| 793 | files, db, err := m.localMarkers(o.Host, rt.DBPath) |
| 794 | if err != nil { |
| 795 | return err |
| 796 | } |
| 797 | if err := moves.PreparedCheck(v.op, *h, files, db); err != nil { |
| 798 | return err |
| 799 | } |
| 800 | default: |
| 801 | if in.NewGeneration == v.op || !claimFreshGeneration(h, in.NewGeneration) || in.Coverage != "" { |
| 802 | return hostbox.ErrChanged |
| 803 | } |
| 804 | } |
| 805 | fresh, err := m.baseline(ctx, o.Host, v, prepared) |
| 806 | if err != nil { |
| 807 | return err |
| 808 | } |
| 809 | if fresh.Generation != baseline.Generation || fresh.MaxTXID != baseline.MaxTXID || !reflect.DeepEqual(fresh.Files, baseline.Files) || !reflect.DeepEqual(fresh.Pin, baseline.Pin) { |
| 810 | return hostbox.ErrChanged |
| 811 | } |
| 812 | return nil |
| 813 | } |
| 814 | |
| 815 | func (m *movesProbeOwners) route(host, op string) (registry.Route, error) { |
| 816 | rt, err := m.Route(host) |
| 817 | if err != nil { |
| 818 | return registry.Route{}, err |
| 819 | } |
| 820 | if rt == nil { |
| 821 | return registry.Route{}, hostbox.ErrChanged |
| 822 | } |
| 823 | if err := heldRoute(*rt, host, op); err != nil { |
| 824 | return registry.Route{}, err |
| 825 | } |
| 826 | return *rt, nil |
| 827 | } |
| 828 | |
| 829 | // admittedProbe checks that o is the admitted probe session in's admission |
| 830 | // made: this box writes the admission's new generation. |
| 831 | func admittedProbe(o hostclaim.Observed, in ClaimLaunchInputs, box string) error { |
| 832 | h := o.Claim.Holder |
| 833 | if h.WriterBox != box || h.Generation != in.Admission.NewGeneration || h.Generation == "" { |
| 834 | return hostbox.ErrChanged |
| 835 | } |
| 836 | return nil |
| 837 | } |
| 838 | |
| 839 | // heldIdentityWork runs its work under the held write's own identity. |
| 840 | type heldIdentityWork struct { |
| 841 | Work hostwrite.Work |
| 842 | ID hostwrite.Identity |
| 843 | } |
| 844 | |
| 845 | func (w heldIdentityWork) With(ctx context.Context, _ hostwrite.Identity, fn func() error) error { |
| 846 | if w.Work == nil { |
| 847 | return hostwrite.ErrUnavailable |
| 848 | } |
| 849 | return w.Work.With(ctx, w.ID, fn) |
| 850 | } |
| 851 | |
| 852 | // Materialize runs before every start of the admitted probe, in the ruled |
| 853 | // order: the held render (its etags compared with the pins before anything |
| 854 | // is selected or written) and the held release under their own selections, |
| 855 | // the launch environment's pin again over the exact env that will run, the |
| 856 | // verified prepared marker under the admitted writer's fence, then the |
| 857 | // start check. The spawn follows only if all of it passed. |
| 858 | func (m *movesProbeOwners) Materialize(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { |
| 859 | if recoveryKind(o) { |
| 860 | return m.recovery().Materialize(ctx, o, in, work) |
| 861 | } |
| 862 | // The captured pin lives as long as this wake: every start attempt |
| 863 | // materializes under the start loop's context, which ends with the |
| 864 | // wake, completed, failed or abandoned. |
| 865 | m.holdEnvPin(ctx, o.Host) |
| 866 | v, err := m.landing(ctx, o) |
| 867 | if err != nil { |
| 868 | return err |
| 869 | } |
| 870 | if err := admittedProbe(o, in, m.Box); err != nil { |
| 871 | return err |
| 872 | } |
| 873 | if work == nil { |
| 874 | return hostwrite.ErrUnavailable |
| 875 | } |
| 876 | // Without the C8 provider every landing wake is refused by name before |
| 877 | // anything is rendered or released (whole-branch fixes, item 1), as the |
| 878 | // recovery owners refuse theirs. |
| 879 | if m.Files == nil { |
| 880 | return claimFilesRefusal() |
| 881 | } |
| 882 | if v.committed { |
| 883 | return m.materializeCommitted(ctx, o, v, in, work) |
| 884 | } |
| 885 | pins := v.rec.Move.Inputs |
| 886 | if in.Identity != launchIdentity(pins) { |
| 887 | return hostbox.ErrChanged |
| 888 | } |
| 889 | fence := writerFence(work, in.Identity) |
| 890 | // The admitted spawn selection still holds under the launch identity. |
| 891 | if err := fence(ctx); err != nil { |
| 892 | return err |
| 893 | } |
| 894 | rt, err := m.route(o.Host, v.op) |
| 895 | if err != nil { |
| 896 | return err |
| 897 | } |
| 898 | if rt.DBPath != in.DBPath { |
| 899 | return hostbox.ErrChanged |
| 900 | } |
| 901 | h := v.holder |
| 902 | captured := hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp} |
| 903 | hr, err := m.Render(ctx, rt, v.op, func(pre appconfig.HeldRender) (hostwrite.Work, error) { |
| 904 | // The pin decides before any selection or write (fix round 1, |
| 905 | // finding 4): a changed configuration is the landing's rollback |
| 906 | // sentence, never the policy's refusal of the write. |
| 907 | if !sameBundle(pre.BundleETags, pins.BundleETags) || pre.OverlayETag != pins.OverlayETag { |
| 908 | return nil, probeInputsChanged() |
| 909 | } |
| 910 | w, err := m.Works(ctx, o.Host, movesActionHeldConfig, captured) |
| 911 | if err != nil { |
| 912 | return nil, err |
| 913 | } |
| 914 | return heldIdentityWork{Work: w, ID: HeldConfigIdentity(pre.BundleETags, pre.OverlayETag)}, nil |
| 915 | }) |
| 916 | if err != nil { |
| 917 | return err |
| 918 | } |
| 919 | if !sameBundle(hr.BundleETags, pins.BundleETags) || hr.OverlayETag != pins.OverlayETag { |
| 920 | return probeInputsChanged() |
| 921 | } |
| 922 | released, err := m.Works(ctx, o.Host, movesActionHeldRelease, captured) |
| 923 | if err != nil { |
| 924 | return err |
| 925 | } |
| 926 | if err := m.Release(ctx, rt, pins.Release, heldIdentityWork{Work: released, ID: hostwrite.Identity{Release: pins.Release}}); err != nil { |
| 927 | return err |
| 928 | } |
| 929 | // The launch Materialize prepares is the one Select pinned: its env's |
| 930 | // pin as captured (captureEnvPin), never recomputed from another read. |
| 931 | if pin, ok := m.capturedPin(o.Host, in.Env); !ok || !moves.CurrentEnvPin(pins.EnvReport) || pin != pins.EnvReport { |
| 932 | return probeInputsChanged() |
| 933 | } |
| 934 | m.recordFence(o.Host, h.Session, work) |
| 935 | receipt, err := m.preparedMarker(ctx, o.Host, v, fence) |
| 936 | if err != nil { |
| 937 | return err |
| 938 | } |
| 939 | if in.Prepared { |
| 940 | m.record(o.Host, h.Session, receipt) |
| 941 | } |
| 942 | if err := m.repairFilesGen(ctx, o, v, work, in.Identity); err != nil { |
| 943 | return err |
| 944 | } |
| 945 | // The admitted generation's files manifest through C8 before the start |
| 946 | // check (whole-branch fixes, item 1): the probe's takes P's prepared |
| 947 | // manifest at the journalled version, stages the env file on disk and |
| 948 | // drops the adoption record; a refusal stops the start. |
| 949 | if err := m.Files.Establish(ctx, o, in, work); err != nil { |
| 950 | return err |
| 951 | } |
| 952 | return m.startCheck(ctx, o, in) |
| 953 | } |
| 954 | |
| 955 | // writerFence is the admitted writer's live fence over its spawn selection: |
| 956 | // the attempt's lease, the session, stamp and generation, and the policy, |
| 957 | // checked afresh on every call (claimLeaseWork over hostbox's selection). |
| 958 | func writerFence(work hostwrite.Work, id hostwrite.Identity) func(context.Context) error { |
| 959 | return func(ctx context.Context) error { |
| 960 | if err := ctx.Err(); err != nil { |
| 961 | return err |
| 962 | } |
| 963 | return work.With(ctx, id, func() error { return nil }) |
| 964 | } |
| 965 | } |
| 966 | |
| 967 | // preparedMarker establishes the verified prepared marker under fence: the |
| 968 | // install.json the build wrote is the journalled manifest, and the location's |
| 969 | // generation.json names this landing (left as it is) or is superseded by the |
| 970 | // move id's marker with the build's pin and that manifest's digest, by |
| 971 | // If-Match on the current version or If-None-Match when there is none. The |
| 972 | // store runs fence before every request attempt. It returns the receipt for |
| 973 | // the move id's marker, or the zero receipt when a later landing generation's |
| 974 | // marker is current. |
| 975 | func (m *movesProbeOwners) preparedMarker(ctx context.Context, host string, v probeLanding, fence func(context.Context) error) (ClaimPublicationReceipt, error) { |
| 976 | h := v.holder |
| 977 | digest, err := manifestDigest(v.manifest) |
| 978 | if err != nil { |
| 979 | return ClaimPublicationReceipt{}, err |
| 980 | } |
| 981 | bounded, cancel := context.WithTimeout(ctx, hostbox.RequestLimit) |
| 982 | defer cancel() |
| 983 | if err := m.verifyInstall(bounded, h, v, digest); err != nil { |
| 984 | return ClaimPublicationReceipt{}, err |
| 985 | } |
| 986 | pin := manifestPin(v.manifest) |
| 987 | root := genselect.File{Key: v.manifest.SnapshotKey, VersionID: v.manifest.SnapshotVersion, Level: 9, MinTXID: 1, MaxTXID: v.manifest.N} |
| 988 | current, err := m.currentMarker(bounded, h) |
| 989 | if err != nil { |
| 990 | return ClaimPublicationReceipt{}, err |
| 991 | } |
| 992 | if current != nil { |
| 993 | marker, err := m.readMarker(bounded, h) |
| 994 | if err != nil { |
| 995 | return ClaimPublicationReceipt{}, err |
| 996 | } |
| 997 | if m.landingMarker(marker, host, v, digest) { |
| 998 | if marker.Op != v.op { |
| 999 | return ClaimPublicationReceipt{}, nil |
| 1000 | } |
| 1001 | return ClaimPublicationReceipt{MarkerVersion: current.ID, Root: root}, nil |
| 1002 | } |
| 1003 | } |
| 1004 | body, err := json.Marshal(genselect.Marker{Op: v.op, Incarnation: h.Incarnation, Host: host, Box: m.Box, InstalledAt: m.Now().UTC(), ManifestSHA256: digest, Pin: pin}) |
| 1005 | if err != nil { |
| 1006 | return ClaimPublicationReceipt{}, err |
| 1007 | } |
| 1008 | expected := "" |
| 1009 | if current != nil { |
| 1010 | expected = current.ETag |
| 1011 | } |
| 1012 | if err := fence(bounded); err != nil { |
| 1013 | return ClaimPublicationReceipt{}, err |
| 1014 | } |
| 1015 | version, err := m.Markers.PutMarker(bounded, m.Bucket, markerKey(h), body, expected, fence) |
| 1016 | if err != nil { |
| 1017 | return ClaimPublicationReceipt{}, err |
| 1018 | } |
| 1019 | if version == "" { |
| 1020 | return ClaimPublicationReceipt{}, hostwrite.ErrUnavailable |
| 1021 | } |
| 1022 | return ClaimPublicationReceipt{MarkerVersion: version, Root: root}, nil |
| 1023 | } |
| 1024 | |
| 1025 | // verifyInstall requires the generation's current install.json to be the |
| 1026 | // journalled manifest, byte for byte (its digest is the marker's). |
| 1027 | func (m *movesProbeOwners) verifyInstall(ctx context.Context, h *hostclaim.Holder, v probeLanding, digest string) error { |
| 1028 | key := moves.InstallManifestKey(moves.GenerationPrefix(h.Location, v.op)) |
| 1029 | versions, err := m.Data.ListVersions(ctx, m.Bucket, key) |
| 1030 | if err != nil { |
| 1031 | return err |
| 1032 | } |
| 1033 | id := "" |
| 1034 | for _, ver := range versions { |
| 1035 | if ver.Key == key && ver.IsLatest && !ver.DeleteMarker { |
| 1036 | id = ver.VersionID |
| 1037 | } |
| 1038 | } |
| 1039 | if id == "" { |
| 1040 | return fmt.Errorf("%w: the generation has no install.json", hostbox.ErrChanged) |
| 1041 | } |
| 1042 | raw, err := m.Data.GetVersion(ctx, m.Bucket, key, id) |
| 1043 | if err != nil { |
| 1044 | return err |
| 1045 | } |
| 1046 | sum := sha256.Sum256(raw) |
| 1047 | if hex.EncodeToString(sum[:]) != digest { |
| 1048 | return fmt.Errorf("%w: install.json is not the journalled manifest", hostbox.ErrChanged) |
| 1049 | } |
| 1050 | return nil |
| 1051 | } |
| 1052 | |
| 1053 | // startCheck is §5.1's check against the claim after the admission's CAS, |
| 1054 | // read fresh: generation.json from this region's location, .files-gen and |
| 1055 | // <dbPath>.gen from this box. |
| 1056 | func (m *movesProbeOwners) startCheck(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { |
| 1057 | fresh, err := hostclaim.Read(ctx, m.Claims, o.Host) |
| 1058 | if err != nil { |
| 1059 | return err |
| 1060 | } |
| 1061 | if !claimSameWriter(o, fresh, hostclaim.SessionContinues) { |
| 1062 | return hostbox.ErrChanged |
| 1063 | } |
| 1064 | marker, err := m.readMarker(ctx, fresh.Claim.Holder) |
| 1065 | if err != nil { |
| 1066 | return err |
| 1067 | } |
| 1068 | files, db, err := m.localMarkers(o.Host, in.DBPath) |
| 1069 | if err != nil { |
| 1070 | return err |
| 1071 | } |
| 1072 | if err := moves.StartCheck(fresh.Claim, marker, files, db); err != nil { |
| 1073 | // A refused start check pages an operator once per op (§13, |
| 1074 | // movespaging.go), and the agent's log says what became of the page. |
| 1075 | op := fresh.Claim.Op.ID |
| 1076 | pagingLogf("moves: %s: %s refused to start %s for %s (%s): %v", o.Host, m.Box, o.Host, op, m.pageStartRefused(ctx, o.Host, op), err) |
| 1077 | return err |
| 1078 | } |
| 1079 | return nil |
| 1080 | } |
| 1081 | |
| 1082 | // readMarker is the location's current generation.json, nil when absent. |
| 1083 | func (m *movesProbeOwners) readMarker(ctx context.Context, h *hostclaim.Holder) (*genselect.Marker, error) { |
| 1084 | current, err := m.currentMarker(ctx, h) |
| 1085 | if err != nil || current == nil { |
| 1086 | return nil, err |
| 1087 | } |
| 1088 | raw, err := m.Data.GetVersion(ctx, m.Bucket, markerKey(h), current.ID) |
| 1089 | if err != nil { |
| 1090 | return nil, err |
| 1091 | } |
| 1092 | var marker genselect.Marker |
| 1093 | if err := json.Unmarshal(raw, &marker); err != nil { |
| 1094 | return nil, fmt.Errorf("%w: %v", moves.ErrStartMarker, err) |
| 1095 | } |
| 1096 | return &marker, nil |
| 1097 | } |
| 1098 | |
| 1099 | // EmptyAuthority: a move never admits the empty bootstrap, a recovery |
| 1100 | // included. |
| 1101 | func (m *movesProbeOwners) EmptyAuthority(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { |
| 1102 | if recoveryKind(o) { |
| 1103 | return m.recovery().EmptyAuthority(ctx, o, in) |
| 1104 | } |
| 1105 | return hostbox.ErrUnsupported |
| 1106 | } |
| 1107 | |
| 1108 | // Publish is Group D's call for a prepared admission, before replication. |
| 1109 | // Materialize has already published the verified prepared marker; this |
| 1110 | // finds it and records the receipt. It writes only under the fence |
| 1111 | // Materialize recorded for the admitted session, and refuses without one. |
| 1112 | func (m *movesProbeOwners) Publish(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { |
| 1113 | if recoveryKind(o) { |
| 1114 | return m.recovery().Publish(ctx, o, in) |
| 1115 | } |
| 1116 | v, err := m.landing(ctx, o) |
| 1117 | if err != nil { |
| 1118 | return err |
| 1119 | } |
| 1120 | if err := admittedProbe(o, in, m.Box); err != nil { |
| 1121 | return err |
| 1122 | } |
| 1123 | h := v.holder |
| 1124 | if !in.Prepared || in.Admission.NewGeneration != v.op { |
| 1125 | return hostbox.ErrChanged |
| 1126 | } |
| 1127 | work := m.fence(o.Host, h.Session) |
| 1128 | if work == nil { |
| 1129 | return fmt.Errorf("%w: no admitted writer's fence for session %d", hostbox.ErrChanged, h.Session) |
| 1130 | } |
| 1131 | receipt, err := m.preparedMarker(ctx, o.Host, v, writerFence(work, in.Identity)) |
| 1132 | if err != nil { |
| 1133 | return err |
| 1134 | } |
| 1135 | if receipt.MarkerVersion == "" { |
| 1136 | return hostbox.ErrChanged |
| 1137 | } |
| 1138 | m.record(o.Host, h.Session, receipt) |
| 1139 | return nil |
| 1140 | } |
| 1141 | |
| 1142 | // PublishFenced writes a restarted attempt's fresh generation marker after |
| 1143 | // the session's own first root, under the publisher's fence, as the claim's |
| 1144 | // serving publisher does (claimServingOwners.PublishFenced). |
| 1145 | func (m *movesProbeOwners) PublishFenced(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, pub ClaimPublication) (ClaimPublicationReceipt, error) { |
| 1146 | if recoveryKind(o) { |
| 1147 | return m.recovery().PublishFenced(ctx, o, in, pub) |
| 1148 | } |
| 1149 | v, err := m.landing(ctx, o) |
| 1150 | if err != nil { |
| 1151 | return ClaimPublicationReceipt{}, err |
| 1152 | } |
| 1153 | if err := admittedProbe(o, in, m.Box); err != nil { |
| 1154 | return ClaimPublicationReceipt{}, err |
| 1155 | } |
| 1156 | if pub.Sync == nil || pub.Check == nil { |
| 1157 | return ClaimPublicationReceipt{}, hostwrite.ErrUnavailable |
| 1158 | } |
| 1159 | h := v.holder |
| 1160 | if in.Prepared || h.Generation == v.op { |
| 1161 | return ClaimPublicationReceipt{}, hostbox.ErrChanged |
| 1162 | } |
| 1163 | // The admitted writer's spawn selection, recorded by Materialize: the |
| 1164 | // fence the local files marker is refreshed under (fix round 1). |
| 1165 | work := m.fence(o.Host, h.Session) |
| 1166 | if work == nil { |
| 1167 | return ClaimPublicationReceipt{}, fmt.Errorf("%w: no admitted writer's fence for session %d", hostbox.ErrChanged, h.Session) |
| 1168 | } |
| 1169 | // The files on disk must be the landing's before anything is published, |
| 1170 | // and their content is recorded as this generation's install record |
| 1171 | // (fix round 3): the digest the refresh, and any later repair, compare. |
| 1172 | var digest string |
| 1173 | if err := work.With(ctx, in.Identity, func() error { |
| 1174 | if err := m.verifyFiles(o.Host, v); err != nil { |
| 1175 | return err |
| 1176 | } |
| 1177 | var err error |
| 1178 | digest, err = m.filesDigest(o.Host) |
| 1179 | return err |
| 1180 | }); err != nil { |
| 1181 | return ClaimPublicationReceipt{}, err |
| 1182 | } |
| 1183 | if err := m.recordFilesDigest(o.Host, v.op, h.Generation, digest); err != nil { |
| 1184 | return ClaimPublicationReceipt{}, err |
| 1185 | } |
| 1186 | return m.publishFresh(ctx, o.Host, h, pub, func() error { return m.refreshFilesGen(ctx, o.Host, v, work, in.Identity, digest) }) |
| 1187 | } |
| 1188 | |
| 1189 | // publishFresh is the fenced publication of the holder's fresh generation, |
| 1190 | // shared by the landing's restarted attempt and the recovery owners: the |
| 1191 | // session's replicator synced, the generation's own first root selected, |
| 1192 | // then generation.json naming the generation superseding the current marker |
| 1193 | // by its exact ETag (created when there is none), pub.Check immediately |
| 1194 | // before the write and before every attempt. A replay that finds this |
| 1195 | // generation's marker from this box records it. refresh runs after the |
| 1196 | // marker, before the receipt is recorded (the local files marker). |
| 1197 | func (m *movesProbeOwners) publishFresh(ctx context.Context, host string, h *hostclaim.Holder, pub ClaimPublication, refresh func() error) (ClaimPublicationReceipt, error) { |
| 1198 | if err := pub.Sync(ctx); err != nil { |
| 1199 | return ClaimPublicationReceipt{}, err |
| 1200 | } |
| 1201 | bounded, cancel := context.WithTimeout(ctx, hostbox.RequestLimit) |
| 1202 | defer cancel() |
| 1203 | plan, err := genselect.Select(bounded, m.Data, m.request(host, h)) |
| 1204 | if err != nil { |
| 1205 | return ClaimPublicationReceipt{}, err |
| 1206 | } |
| 1207 | // Only this session's bound replica client writes under the fresh |
| 1208 | // generation, so its root is the session's own acknowledged upload. |
| 1209 | if plan.Generation != h.Generation || plan.Root == nil || plan.Root.MinTXID != 1 || plan.Root.VersionID == "" { |
| 1210 | return ClaimPublicationReceipt{}, hostbox.ErrChanged |
| 1211 | } |
| 1212 | root := *plan.Root |
| 1213 | current, err := m.currentMarker(bounded, h) |
| 1214 | if err != nil { |
| 1215 | return ClaimPublicationReceipt{}, err |
| 1216 | } |
| 1217 | if current != nil && plan.Marker != nil && plan.MarkerVersion == current.ID && plan.Marker.Op == h.Generation { |
| 1218 | if plan.Marker.Box != m.Box || plan.Marker.Incarnation != h.Incarnation { |
| 1219 | return ClaimPublicationReceipt{}, hostbox.ErrChanged |
| 1220 | } |
| 1221 | receipt := ClaimPublicationReceipt{MarkerVersion: current.ID, Root: root} |
| 1222 | if err := refresh(); err != nil { |
| 1223 | return ClaimPublicationReceipt{}, err |
| 1224 | } |
| 1225 | m.record(host, h.Session, receipt) |
| 1226 | return receipt, nil |
| 1227 | } |
| 1228 | body, err := json.Marshal(genselect.Marker{Op: h.Generation, Incarnation: h.Incarnation, Host: host, Box: m.Box, InstalledAt: m.Now().UTC()}) |
| 1229 | if err != nil { |
| 1230 | return ClaimPublicationReceipt{}, err |
| 1231 | } |
| 1232 | expected := "" |
| 1233 | if current != nil { |
| 1234 | expected = current.ETag |
| 1235 | } |
| 1236 | if err := pub.Check(bounded); err != nil { |
| 1237 | return ClaimPublicationReceipt{}, err |
| 1238 | } |
| 1239 | version, err := m.Markers.PutMarker(bounded, m.Bucket, markerKey(h), body, expected, pub.Check) |
| 1240 | if err != nil { |
| 1241 | return ClaimPublicationReceipt{}, err |
| 1242 | } |
| 1243 | if version == "" { |
| 1244 | return ClaimPublicationReceipt{}, hostwrite.ErrUnavailable |
| 1245 | } |
| 1246 | receipt := ClaimPublicationReceipt{MarkerVersion: version, Root: root} |
| 1247 | if err := refresh(); err != nil { |
| 1248 | return ClaimPublicationReceipt{}, err |
| 1249 | } |
| 1250 | m.record(host, h.Session, receipt) |
| 1251 | return receipt, nil |
| 1252 | } |
| 1253 | |
| 1254 | // recordFence keeps the admitted spawn selection Materialize ran under, by |
| 1255 | // host and session: the writer's fence for that session's marker writes and |
| 1256 | // its files marker refresh. |
| 1257 | func (m *movesProbeOwners) recordFence(host string, session uint64, work hostwrite.Work) { |
| 1258 | m.mu.Lock() |
| 1259 | defer m.mu.Unlock() |
| 1260 | if m.fences == nil { |
| 1261 | m.fences = map[string]hostwrite.Work{} |
| 1262 | } |
| 1263 | m.fences[claimPublishedKey(host, session)] = work |
| 1264 | } |
| 1265 | |
| 1266 | // fence is the admitted spawn selection Materialize recorded for the |
| 1267 | // session, or nil. |
| 1268 | func (m *movesProbeOwners) fence(host string, session uint64) hostwrite.Work { |
| 1269 | m.mu.Lock() |
| 1270 | defer m.mu.Unlock() |
| 1271 | return m.fences[claimPublishedKey(host, session)] |
| 1272 | } |
| 1273 | |
| 1274 | // recovery is the recovery owners over this owner set's sources. |
| 1275 | func (m *movesProbeOwners) recovery() movesRecoveryOwners { return movesRecoveryOwners{m: m} } |
| 1276 | |
| 1277 | // verifyFiles requires the files directory and <host>.env on disk to be the |
| 1278 | // ones the landing placed: the install receipt's identities |
| 1279 | // (landStepFilesPlaced). Anything else fails the publication. |
| 1280 | func (m *movesProbeOwners) verifyFiles(host string, v probeLanding) error { |
| 1281 | j, err := moves.ReadJournal(m.DataDir, host, v.op) |
| 1282 | if err != nil { |
| 1283 | return err |
| 1284 | } |
| 1285 | placed, err := journalledStaged(j, landStepFilesPlaced) |
| 1286 | if err != nil { |
| 1287 | return fmt.Errorf("%w: %v", hostbox.ErrChanged, err) |
| 1288 | } |
| 1289 | dir, env := m.FilesDir(host) |
| 1290 | if ok, err := sameIdentity(dir, placed.Files); err != nil || !ok { |
| 1291 | return fmt.Errorf("%w: %s is not the files directory the landing placed (%v)", hostbox.ErrChanged, dir, err) |
| 1292 | } |
| 1293 | if placed.Env != "" { |
| 1294 | if ok, err := sameIdentity(env, placed.Env); err != nil || !ok { |
| 1295 | return fmt.Errorf("%w: %s is not the environment file the landing placed (%v)", hostbox.ErrChanged, env, err) |
| 1296 | } |
| 1297 | } |
| 1298 | return nil |
| 1299 | } |
| 1300 | |
| 1301 | // refreshFilesGen points .files-gen at the published generation (fix round |
| 1302 | // 1, finding 1), inside the admitted writer's fence, after the files on disk |
| 1303 | // are proved the landing's again: the files are the same bytes the move |
| 1304 | // installed, and the generation that now owns them is the one just |
| 1305 | // published. Task 4's start check is unchanged: the next admission's cold |
| 1306 | // predecessor is this generation. <dbPath>.gen needs no refresh: every cold |
| 1307 | // restore rewrites it to the admitted generation (H11). |
| 1308 | func (m *movesProbeOwners) refreshFilesGen(ctx context.Context, host string, v probeLanding, work hostwrite.Work, id hostwrite.Identity, digest string) error { |
| 1309 | h := v.holder |
| 1310 | marker, err := json.Marshal(moves.FilesGen{Op: h.Generation, Incarnation: h.Incarnation}) |
| 1311 | if err != nil { |
| 1312 | return err |
| 1313 | } |
| 1314 | dir, _ := m.FilesDir(host) |
| 1315 | return work.With(ctx, id, func() error { |
| 1316 | if err := m.verifyFilesContent(host, v, digest); err != nil { |
| 1317 | return err |
| 1318 | } |
| 1319 | if err := syncPath(dir); err != nil { |
| 1320 | return err |
| 1321 | } |
| 1322 | return writeFileAtomic(filepath.Join(dir, archive.MarkerName), marker, 0o644) |
| 1323 | }) |
| 1324 | } |
| 1325 | |
| 1326 | // filesPlainStep is the op journal step a published generation's files |
| 1327 | // digest is recorded under: its install record (fix round 3). |
| 1328 | const filesPlainStep = "files_plain_" |
| 1329 | |
| 1330 | // filesDigest is the files tree's plain digest, the landing's own measure |
| 1331 | // (archive.PlainDigest over the files directory and <host>.env; .files-gen |
| 1332 | // is not part of it). The tree is capped at 256 MiB. |
| 1333 | func (m *movesProbeOwners) filesDigest(host string) (string, error) { |
| 1334 | dir, env := m.FilesDir(host) |
| 1335 | plain, _, _, err := archivePlainDigest(dir, env) |
| 1336 | if err != nil { |
| 1337 | return "", err |
| 1338 | } |
| 1339 | if plain == "" { |
| 1340 | return "", fmt.Errorf("%w: the files tree has no digest", hostbox.ErrChanged) |
| 1341 | } |
| 1342 | return plain, nil |
| 1343 | } |
| 1344 | |
| 1345 | // recordFilesDigest journals generation g's files digest, the install record |
| 1346 | // a later repair compares. A different digest already recorded for g is a |
| 1347 | // conflict. |
| 1348 | func (m *movesProbeOwners) recordFilesDigest(host, op, g, digest string) error { |
| 1349 | m.mu.Lock() |
| 1350 | defer m.mu.Unlock() |
| 1351 | // Under the journal's one lock (moves.UpdateJournal): the landing's pass |
| 1352 | // and the paging rules write this journal too. |
| 1353 | err := moves.UpdateJournal(m.DataDir, host, op, func(j *moves.Journal) error { |
| 1354 | if have, ok := j.Steps[filesPlainStep+g]; ok { |
| 1355 | if have != digest { |
| 1356 | return fmt.Errorf("%w: generation %s recorded other files", hostbox.ErrChanged, g) |
| 1357 | } |
| 1358 | return errJournalUnchanged |
| 1359 | } |
| 1360 | j.Mark(filesPlainStep+g, digest) |
| 1361 | return nil |
| 1362 | }) |
| 1363 | if errors.Is(err, errJournalUnchanged) { |
| 1364 | return nil |
| 1365 | } |
| 1366 | return err |
| 1367 | } |
| 1368 | |
| 1369 | // errJournalUnchanged ends a moves.UpdateJournal that has nothing to write; |
| 1370 | // its caller treats it as success. |
| 1371 | var errJournalUnchanged = errors.New("moves journal: nothing to write") |
| 1372 | |
| 1373 | // recordedFilesDigest is generation g's recorded files digest: the move id's |
| 1374 | // is the source's stopped baseline the landing installed (Move.Source.Files); |
| 1375 | // a later generation's is the one its publication journalled. false when |
| 1376 | // none is recorded. |
| 1377 | func (m *movesProbeOwners) recordedFilesDigest(host string, v probeLanding, g string) (string, bool) { |
| 1378 | if g == v.op { |
| 1379 | src := v.rec.Move.Source |
| 1380 | if src == nil || src.Files == nil || src.Files.Plain == "" { |
| 1381 | return "", false |
| 1382 | } |
| 1383 | return src.Files.Plain, true |
| 1384 | } |
| 1385 | j, err := moves.ReadJournal(m.DataDir, host, v.op) |
| 1386 | if err != nil { |
| 1387 | return "", false |
| 1388 | } |
| 1389 | d, ok := j.Steps[filesPlainStep+g] |
| 1390 | return d, ok && d != "" |
| 1391 | } |
| 1392 | |
| 1393 | // verifyFilesContent requires both the landing's placement (identity) and |
| 1394 | // the recorded content (the plain digest): never identity alone. |
| 1395 | func (m *movesProbeOwners) verifyFilesContent(host string, v probeLanding, want string) error { |
| 1396 | if want == "" { |
| 1397 | return fmt.Errorf("%w: no recorded files digest", hostbox.ErrChanged) |
| 1398 | } |
| 1399 | if err := m.verifyFiles(host, v); err != nil { |
| 1400 | return err |
| 1401 | } |
| 1402 | got, err := m.filesDigest(host) |
| 1403 | if err != nil { |
| 1404 | return err |
| 1405 | } |
| 1406 | if got != want { |
| 1407 | return fmt.Errorf("%w: the files on disk are not the recorded ones", hostbox.ErrChanged) |
| 1408 | } |
| 1409 | return nil |
| 1410 | } |
| 1411 | |
| 1412 | func (m *movesProbeOwners) record(host string, session uint64, receipt ClaimPublicationReceipt) { |
| 1413 | m.mu.Lock() |
| 1414 | defer m.mu.Unlock() |
| 1415 | if m.published == nil { |
| 1416 | m.published = map[string]ClaimPublicationReceipt{} |
| 1417 | } |
| 1418 | m.published[claimPublishedKey(host, session)] = receipt |
| 1419 | } |
| 1420 | |
| 1421 | // PublicationEvidence requires this process's receipt for the admitted |
| 1422 | // session: the current marker is exactly the version it wrote, naming the |
| 1423 | // holder's generation, incarnation, host and this box (and, for the move |
| 1424 | // id, the build's pin), and the root it named still exists, current. |
| 1425 | func (m *movesProbeOwners) PublicationEvidence(ctx context.Context, o hostclaim.Observed) error { |
| 1426 | if recoveryKind(o) { |
| 1427 | return m.recovery().PublicationEvidence(ctx, o) |
| 1428 | } |
| 1429 | v, err := m.landing(ctx, o) |
| 1430 | if err != nil { |
| 1431 | return err |
| 1432 | } |
| 1433 | return m.publicationEvidence(ctx, o.Host, v.holder, v.op, v.manifest) |
| 1434 | } |
| 1435 | |
| 1436 | // publicationEvidence is PublicationEvidence's check of the admitted |
| 1437 | // session's receipt, shared with the recovery owners. prepared names the |
| 1438 | // prepared generation whose marker also carries manifest's pin and digest |
| 1439 | // ("" for none). |
| 1440 | func (m *movesProbeOwners) publicationEvidence(ctx context.Context, host string, h *hostclaim.Holder, prepared string, manifest moves.InstallManifest) error { |
| 1441 | if h.WriterBox != m.Box { |
| 1442 | return hostbox.ErrChanged |
| 1443 | } |
| 1444 | m.mu.Lock() |
| 1445 | receipt, ok := m.published[claimPublishedKey(host, h.Session)] |
| 1446 | m.mu.Unlock() |
| 1447 | prefix, err := hostbox.GenerationPrefix(*h) |
| 1448 | if !ok || err != nil || receipt.MarkerVersion == "" || receipt.Root.VersionID == "" || !strings.HasPrefix(receipt.Root.Key, prefix) { |
| 1449 | return hostbox.ErrChanged |
| 1450 | } |
| 1451 | plan, err := genselect.Select(ctx, m.Data, m.request(host, h)) |
| 1452 | if err != nil { |
| 1453 | return err |
| 1454 | } |
| 1455 | if plan.Generation != h.Generation || plan.Marker == nil || plan.MarkerVersion != receipt.MarkerVersion || plan.Marker.Op != h.Generation || plan.Marker.Box != m.Box || plan.Marker.Incarnation != h.Incarnation || plan.Marker.Host != host { |
| 1456 | return hostbox.ErrChanged |
| 1457 | } |
| 1458 | if prepared != "" && h.Generation == prepared { |
| 1459 | digest, err := manifestDigest(manifest) |
| 1460 | if err != nil { |
| 1461 | return err |
| 1462 | } |
| 1463 | if plan.Marker.Pin != manifestPin(manifest) || plan.Marker.ManifestSHA256 != digest { |
| 1464 | return hostbox.ErrChanged |
| 1465 | } |
| 1466 | } |
| 1467 | versions, err := m.Data.ListVersions(ctx, m.Bucket, receipt.Root.Key) |
| 1468 | if err != nil { |
| 1469 | return err |
| 1470 | } |
| 1471 | exact, current := false, false |
| 1472 | for _, ver := range versions { |
| 1473 | if ver.Key != receipt.Root.Key || ver.DeleteMarker { |
| 1474 | continue |
| 1475 | } |
| 1476 | exact = exact || ver.VersionID == receipt.Root.VersionID |
| 1477 | current = current || ver.IsLatest |
| 1478 | } |
| 1479 | if !exact || !current { |
| 1480 | return hostbox.ErrChanged |
| 1481 | } |
| 1482 | return nil |
| 1483 | } |
| 1484 | |
| 1485 | // The committed wake (Plan 3 Task 7; Task 5's concern). After the commit the |
| 1486 | // claim stays landing/committed until the source's finalization closes the |
| 1487 | // move, and Group D admits a landing wake there (K6's committed row). A wake |
| 1488 | // happens only when no process serves the host: the probe's own entry, while |
| 1489 | // it lives, is the serving process and is never restarted. The wake is a |
| 1490 | // cold admission into a fresh generation from the certified remote (the |
| 1491 | // landing's published marker, genselect.Select), continuing the admitted |
| 1492 | // session's generation: no kept stamp, no coverage. It launches the row's |
| 1493 | // ordinary launch (the claim's launch snapshot, its own identity, which |
| 1494 | // MovesClaimPolicyWithInputs's committed check accepts), never the probe |
| 1495 | // variable, and the held inputs are not rendered again: after the open the |
| 1496 | // row is config sync's. |
| 1497 | |
| 1498 | // committedRoute is the row a committed wake runs: this host's, held by the |
| 1499 | // move until the open, then ready. |
| 1500 | func committedRoute(rt registry.Route, host, op string) error { |
| 1501 | if rt.Host != host || !filepath.IsAbs(rt.DBPath) { |
| 1502 | return hostbox.ErrChanged |
| 1503 | } |
| 1504 | if (registry.IsHold(rt.Status) && rt.HoldOp == op) || (rt.Status == "ready" && rt.HoldOp == "") { |
| 1505 | return nil |
| 1506 | } |
| 1507 | return hostbox.ErrChanged |
| 1508 | } |
| 1509 | |
| 1510 | // withoutProbeVars is a launch environment with the probe-only variables |
| 1511 | // removed. |
| 1512 | func withoutProbeVars(env []string) []string { |
| 1513 | out := make([]string, 0, len(env)) |
| 1514 | for _, kv := range env { |
| 1515 | if k, _, _ := strings.Cut(kv, "="); !probeOnlyVars[k] { |
| 1516 | out = append(out, kv) |
| 1517 | } |
| 1518 | } |
| 1519 | return out |
| 1520 | } |
| 1521 | |
| 1522 | func (m *movesProbeOwners) committedAdmission(o hostclaim.Observed, v probeLanding, gen string, plan *genselect.Plan) (hostclaim.Admission, error) { |
| 1523 | h := v.holder |
| 1524 | predecessor, err := predecessorRegion(h, v.rec.Move.From) |
| 1525 | if err != nil { |
| 1526 | return hostclaim.Admission{}, err |
| 1527 | } |
| 1528 | return hostclaim.Admission{ObservedOpID: v.op, ObservedPhase: o.Claim.Op.Phase, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: m.Box, Mode: "cold", |
| 1529 | NewGeneration: gen, KeepStamp: h.Stamp > h.Session, Region: m.Region, PredecessorRegion: predecessor, RemoteMaxTXID: plan.MaxTXID}, nil |
| 1530 | } |
| 1531 | |
| 1532 | func (m *movesProbeOwners) selectCommitted(ctx context.Context, rt registry.Route, o hostclaim.Observed, v probeLanding, deadline time.Time) (ClaimLaunchInputs, error) { |
| 1533 | if err := committedRoute(rt, o.Host, v.op); err != nil { |
| 1534 | return ClaimLaunchInputs{}, err |
| 1535 | } |
| 1536 | h := v.holder |
| 1537 | if h.WriterBox != m.Box { |
| 1538 | return ClaimLaunchInputs{}, hostbox.ErrChanged |
| 1539 | } |
| 1540 | gen := m.NewID() |
| 1541 | if gen == v.op || !claimFreshGeneration(h, gen) { |
| 1542 | return ClaimLaunchInputs{}, hostwrite.ErrUnavailable |
| 1543 | } |
| 1544 | plan, err := genselect.Select(ctx, m.Data, m.request(o.Host, h)) |
| 1545 | if err != nil { |
| 1546 | return ClaimLaunchInputs{}, err |
| 1547 | } |
| 1548 | l, report, err := m.Launch(ctx, rt) |
| 1549 | if err != nil { |
| 1550 | return ClaimLaunchInputs{}, err |
| 1551 | } |
| 1552 | if !filepath.IsAbs(l.Executable) || !claimHexDigest(l.ExecutableSHA256) || l.Version == "" { |
| 1553 | return ClaimLaunchInputs{}, hostwrite.ErrUnavailable |
| 1554 | } |
| 1555 | // The committed wake runs the move's pinned inputs (InputPolicy), judged |
| 1556 | // on the launch itself: its environment's report and its release. |
| 1557 | pins := v.rec.Move.Inputs |
| 1558 | id := committedLaunchIdentity(report.Digest, l.Version) |
| 1559 | if !moves.CurrentEnvPin(pins.EnvReport) || id != pinnedLaunchIdentity(pins) { |
| 1560 | return ClaimLaunchInputs{}, fmt.Errorf("%w: the launch is not the move's pinned inputs", hostbox.ErrChanged) |
| 1561 | } |
| 1562 | admission, err := m.committedAdmission(o, v, gen, plan) |
| 1563 | if err != nil { |
| 1564 | return ClaimLaunchInputs{}, err |
| 1565 | } |
| 1566 | in := ClaimLaunchInputs{Roles: []string{"main"}, Identity: id, Baseline: plan, Admission: admission, |
| 1567 | Executable: l.Executable, ExecutableSHA256: l.ExecutableSHA256, Args: append([]string(nil), l.Args...), Env: withoutProbeVars(l.Env), |
| 1568 | DBPath: rt.DBPath, Version: l.Version, Staged: l.Staged, Limit: l.Limit} |
| 1569 | m.captureEnvPin(rt.Host, in.Env, report.Digest, deadline) |
| 1570 | return in, nil |
| 1571 | } |
| 1572 | |
| 1573 | func (m *movesProbeOwners) committedEvidence(ctx context.Context, o hostclaim.Observed, v probeLanding, in hostclaim.Admission, baseline *genselect.Plan) error { |
| 1574 | h := v.holder |
| 1575 | if baseline == nil || h.WriterBox != m.Box || in.NewGeneration == v.op || !claimFreshGeneration(h, in.NewGeneration) || in.PredecessorCertified || in.LossID != "" || len(in.AdditionalWritesLost) != 0 { |
| 1576 | return hostbox.ErrChanged |
| 1577 | } |
| 1578 | want, err := m.committedAdmission(o, v, in.NewGeneration, baseline) |
| 1579 | if err != nil { |
| 1580 | return err |
| 1581 | } |
| 1582 | if !reflect.DeepEqual(in, want) { |
| 1583 | return hostbox.ErrChanged |
| 1584 | } |
| 1585 | fresh, err := genselect.Select(ctx, m.Data, m.request(o.Host, h)) |
| 1586 | if err != nil { |
| 1587 | return err |
| 1588 | } |
| 1589 | if fresh.Generation != baseline.Generation || fresh.MaxTXID != baseline.MaxTXID || !reflect.DeepEqual(fresh.Files, baseline.Files) || !reflect.DeepEqual(fresh.Pin, baseline.Pin) { |
| 1590 | return hostbox.ErrChanged |
| 1591 | } |
| 1592 | return nil |
| 1593 | } |
| 1594 | |
| 1595 | func (m *movesProbeOwners) materializeCommitted(ctx context.Context, o hostclaim.Observed, v probeLanding, in ClaimLaunchInputs, work hostwrite.Work) error { |
| 1596 | // The identity is the launch that will run (in.Env, in.Version), and it |
| 1597 | // is the pins'. |
| 1598 | pin, ok := m.capturedPin(o.Host, in.Env) |
| 1599 | if !ok || in.Identity != committedLaunchIdentity(pin, in.Version) || in.Identity != pinnedLaunchIdentity(v.rec.Move.Inputs) { |
| 1600 | return hostbox.ErrChanged |
| 1601 | } |
| 1602 | if err := writerFence(work, in.Identity)(ctx); err != nil { |
| 1603 | return err |
| 1604 | } |
| 1605 | m.recordFence(o.Host, v.holder.Session, work) |
| 1606 | rt, err := m.Route(o.Host) |
| 1607 | if err != nil { |
| 1608 | return err |
| 1609 | } |
| 1610 | if rt == nil || rt.DBPath != in.DBPath { |
| 1611 | return hostbox.ErrChanged |
| 1612 | } |
| 1613 | if err := committedRoute(*rt, o.Host, v.op); err != nil { |
| 1614 | return err |
| 1615 | } |
| 1616 | if err := m.repairFilesGen(ctx, o, v, work, in.Identity); err != nil { |
| 1617 | return err |
| 1618 | } |
| 1619 | // The fresh generation's files manifest through C8, copied from the |
| 1620 | // admitted predecessor's, before the start check (item 1). |
| 1621 | if err := m.Files.Establish(ctx, o, in, work); err != nil { |
| 1622 | return err |
| 1623 | } |
| 1624 | return m.startCheck(ctx, o, in) |
| 1625 | } |
| 1626 | |
| 1627 | // InputPolicy is the moves owner's input policy: the entry point claim Plan |
| 1628 | // 10 rev 5's ClaimOwnerDispatch routes a move kind's input check to (the |
| 1629 | // shape of ClaimOwnerBindings.InputPolicy; a move kind never falls back to |
| 1630 | // serving's inputs). It answers the committed wake's spawn (Plan 3 Task 7): |
| 1631 | // at landing/committed the launch identity, computed by the owner from the |
| 1632 | // launch that will run (committedLaunchIdentity: its environment's report |
| 1633 | // digest and its release), must be the move's pins' (Move.Inputs.EnvReport |
| 1634 | // and Move.Inputs.Release), over a committed record of this move. The |
| 1635 | // other move-kind inputs are MovesClaimPolicy's own rows; anything else is |
| 1636 | // refused here. |
| 1637 | func (m *movesProbeOwners) InputPolicy(ctx context.Context, c *hostclaim.Claim, s hostbox.Selection, id hostwrite.Identity) error { |
| 1638 | if m == nil || m.Records == nil || ctx == nil { |
| 1639 | return hostwrite.ErrUnavailable |
| 1640 | } |
| 1641 | if c == nil || c.Holder == nil || c.Op.Kind != hostclaim.KindLanding || c.Op.Phase != hostclaim.PhaseLandingCommitted || |
| 1642 | s.Phase != hostclaim.PhaseLandingCommitted || s.Action != movesActionSpawn || s.Op != c.Op.ID || s.Host != c.Host { |
| 1643 | return hostbox.ErrChanged |
| 1644 | } |
| 1645 | rec, err := m.Records(ctx, c.Holder.Account, c.Holder.App, c.Host) |
| 1646 | if err != nil { |
| 1647 | return err |
| 1648 | } |
| 1649 | if rec == nil || rec.Host != c.Host || rec.Move == nil || rec.Move.ID != c.Op.ID || rec.Move.Incarnation != c.Holder.Incarnation || |
| 1650 | rec.Move.State != provision.MoveCommitted || rec.Move.Inputs == nil || rec.Move.Inputs.Release == "" { |
| 1651 | return hostbox.ErrChanged |
| 1652 | } |
| 1653 | if !moves.CurrentEnvPin(rec.Move.Inputs.EnvReport) || id != pinnedLaunchIdentity(rec.Move.Inputs) { |
| 1654 | return fmt.Errorf("%w: the committed launch is not the move's pinned inputs", hostbox.ErrChanged) |
| 1655 | } |
| 1656 | return nil |
| 1657 | } |
| 1658 | |
| 1659 | // committedEnvPrefix marks a committed launch's config identity: the report |
| 1660 | // digest of its environment. |
| 1661 | const committedEnvPrefix = "launch-env:" |
| 1662 | |
| 1663 | // committedLaunchIdentity is the committed wake's identity, computed from |
| 1664 | // the launch that will run (fix round 2, item 2): its environment's |
| 1665 | // move-portable pin as captured with it (Launch's report, CARLOS_MOVE_PROBE |
| 1666 | // excluded) and its release. The input policy compares it with the pins. |
| 1667 | func committedLaunchIdentity(pin, version string) hostwrite.Identity { |
| 1668 | return hostwrite.Identity{Config: committedEnvPrefix + pin, Release: version} |
| 1669 | } |
| 1670 | |
| 1671 | // pinnedLaunchIdentity is the identity a launch matching the move's pins |
| 1672 | // carries. |
| 1673 | func pinnedLaunchIdentity(in *provision.MoveInputs) hostwrite.Identity { |
| 1674 | return hostwrite.Identity{Config: committedEnvPrefix + in.EnvReport, Release: in.Release} |
| 1675 | } |
| 1676 | |
| 1677 | // repairFilesGen brings .files-gen to the admitted generation's cold |
| 1678 | // predecessor T before the start check, inside the admitted writer's fence, |
| 1679 | // in two cases, each verified by content (the files tree's plain digest |
| 1680 | // against a recorded one, with the landing's placement), never by identity |
| 1681 | // alone: |
| 1682 | // |
| 1683 | // - an interrupted refresh (Task 7 fix round 2): the published marker |
| 1684 | // names T itself, and .files-gen names an older generation of T's |
| 1685 | // ancestry; the files must match T's recorded digest. |
| 1686 | // - unpublished cold admissions (final review, ruling 1): the published |
| 1687 | // marker names an older generation P, and T reaches P through cold |
| 1688 | // admissions on this box and region, none of which published (a |
| 1689 | // publication would have superseded P's marker); .files-gen names P or |
| 1690 | // one of those admissions. The files must match P's recorded digest: |
| 1691 | // every admission since P restored from P's data and its files are P's. |
| 1692 | // |
| 1693 | // Anything else is left alone, for the unchanged start check to decide. |
| 1694 | func (m *movesProbeOwners) repairFilesGen(ctx context.Context, o hostclaim.Observed, v probeLanding, work hostwrite.Work, id hostwrite.Identity) error { |
| 1695 | h := o.Claim.Holder |
| 1696 | marker, err := m.readMarker(ctx, h) |
| 1697 | if err != nil || marker == nil { |
| 1698 | return err |
| 1699 | } |
| 1700 | published := marker.Op |
| 1701 | if published == "" || marker.Incarnation != h.Incarnation || marker.Host != o.Host { |
| 1702 | return nil |
| 1703 | } |
| 1704 | t, ok := coldPredecessor(h, h.Generation) |
| 1705 | if !ok || t == "" { |
| 1706 | return nil |
| 1707 | } |
| 1708 | dir, _ := m.FilesDir(o.Host) |
| 1709 | files, _, err := m.localMarkers(o.Host, filepath.Join(dir, ".none")) |
| 1710 | if err != nil || files == nil || files.Op == t || files.Incarnation != h.Incarnation { |
| 1711 | return nil |
| 1712 | } |
| 1713 | var vouch string // the published generation whose recorded digest the files must match |
| 1714 | switch { |
| 1715 | case published == t: |
| 1716 | if !ancestorOf(h, t, files.Op) { |
| 1717 | return nil |
| 1718 | } |
| 1719 | vouch = t |
| 1720 | default: |
| 1721 | path, ok := m.unpublishedPath(h, t, published) |
| 1722 | if !ok || !containsString(path[1:], files.Op) { |
| 1723 | return nil |
| 1724 | } |
| 1725 | vouch = published |
| 1726 | } |
| 1727 | want, ok := m.recordedFilesDigest(o.Host, v, vouch) |
| 1728 | if !ok { |
| 1729 | return nil // nothing recorded: the start check decides |
| 1730 | } |
| 1731 | body, err := json.Marshal(moves.FilesGen{Op: t, Incarnation: h.Incarnation}) |
| 1732 | if err != nil { |
| 1733 | return err |
| 1734 | } |
| 1735 | return work.With(ctx, id, func() error { |
| 1736 | if m.verifyFilesContent(o.Host, v, want) != nil { |
| 1737 | return nil // not the vouching generation's files: the start check decides |
| 1738 | } |
| 1739 | if err := syncPath(dir); err != nil { |
| 1740 | return err |
| 1741 | } |
| 1742 | log.Printf("moves: %s: .files-gen repaired from %s to %s (vouched by the published %s)", o.Host, files.Op, t, vouch) |
| 1743 | return writeFileAtomic(filepath.Join(dir, archive.MarkerName), body, 0o644) |
| 1744 | }) |
| 1745 | } |
| 1746 | |
| 1747 | // unpublishedPath walks from back to the published generation to through |
| 1748 | // cold admissions, each into the generation walked and on this box and |
| 1749 | // region. It returns the generations from from to to, inclusive, or false |
| 1750 | // when the chain leaves this box or region, breaks, loops, or never |
| 1751 | // reaches to. |
| 1752 | func (m *movesProbeOwners) unpublishedPath(h *hostclaim.Holder, from, to string) ([]string, bool) { |
| 1753 | path := []string{from} |
| 1754 | for g := from; g != to; { |
| 1755 | if len(path) > len(h.Admissions)+1 { |
| 1756 | return nil, false |
| 1757 | } |
| 1758 | var found *hostclaim.AdmissionRecord |
| 1759 | for i := len(h.Admissions) - 1; i >= 0; i-- { |
| 1760 | if a := h.Admissions[i]; a.Generation == g && a.Mode == "cold" { |
| 1761 | found = &h.Admissions[i] |
| 1762 | break |
| 1763 | } |
| 1764 | } |
| 1765 | if found == nil || found.Box != m.Box || !park.SameRegion(m.HomeRegion, found.Region, m.Region) || found.Predecessor == "" { |
| 1766 | return nil, false |
| 1767 | } |
| 1768 | g = found.Predecessor |
| 1769 | path = append(path, g) |
| 1770 | } |
| 1771 | return path, true |
| 1772 | } |
| 1773 | |
| 1774 | // coldPredecessor is the predecessor of the newest cold admission into g. |
| 1775 | func coldPredecessor(h *hostclaim.Holder, g string) (string, bool) { |
| 1776 | for i := len(h.Admissions) - 1; i >= 0; i-- { |
| 1777 | if a := h.Admissions[i]; a.Generation == g && a.Mode == "cold" { |
| 1778 | return a.Predecessor, true |
| 1779 | } |
| 1780 | } |
| 1781 | return "", false |
| 1782 | } |
| 1783 | |
| 1784 | // ancestorOf reports whether old is reached by walking g's cold |
| 1785 | // predecessors in the holder's admissions. |
| 1786 | func ancestorOf(h *hostclaim.Holder, g, old string) bool { |
| 1787 | for steps := 0; steps <= len(h.Admissions); steps++ { |
| 1788 | pred, ok := coldPredecessor(h, g) |
| 1789 | if !ok || pred == "" { |
| 1790 | return false |
| 1791 | } |
| 1792 | if pred == old { |
| 1793 | return true |
| 1794 | } |
| 1795 | g = pred |
| 1796 | } |
| 1797 | return false |
| 1798 | } |
| 1799 | |
amadan