| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "io/fs" |
| 9 | "path/filepath" |
| 10 | "reflect" |
| 11 | "strconv" |
| 12 | "strings" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/activator" |
| 15 | "github.com/carlosframework/platform/internal/appconfig" |
| 16 | "github.com/carlosframework/platform/internal/genselect" |
| 17 | "github.com/carlosframework/platform/internal/hostbox" |
| 18 | "github.com/carlosframework/platform/internal/hostclaim" |
| 19 | "github.com/carlosframework/platform/internal/hostguard" |
| 20 | "github.com/carlosframework/platform/internal/hostwrite" |
| 21 | "github.com/carlosframework/platform/internal/moves" |
| 22 | "github.com/carlosframework/platform/internal/moves/archive" |
| 23 | "github.com/carlosframework/platform/internal/park" |
| 24 | "github.com/carlosframework/platform/internal/provision" |
| 25 | "github.com/carlosframework/platform/internal/registry" |
| 26 | ) |
| 27 | |
| 28 | // The moves owners for a recovery probe (moves Phase 1 Plan 4 Task 9; |
| 29 | // rulings Q7 and Q6 of the moves-after-Plan-12 preflight, J2, J14, H14). The |
| 30 | // moves owner set (movesProbeOwners) dispatches by the observed op's kind: |
| 31 | // a landing to the landing probe's owners, restoring and reverting here. |
| 32 | // |
| 33 | // One shape is served: the source's recovery probe, the claim restoring (a |
| 34 | // rollback's return) or reverting (rev_phase probing) at op.phase probing, |
| 35 | // this box executing as the move's source in the move's source region, not |
| 36 | // the target region and not lost; the record this move and this holder |
| 37 | // incarnation, rolling_back for the claim's rollback at source phase probing |
| 38 | // or reverting and not reopened; the row held by the rollback id (restoring) |
| 39 | // or the move id (reverting). Everything else is refused before any lease, |
| 40 | // stop or launch. |
| 41 | // |
| 42 | // Every recovery admission is cold and keeps no stamp (J14: S is retired by |
| 43 | // the first recovery admission). The first one is told by the holder still |
| 44 | // at the source's stamp S (op.journal.source_stamp, which ReturnToSource |
| 45 | // and Revert leave in holder.stamp); any later one is a restart. |
| 46 | // |
| 47 | // - Select (read-only). A dirty return's first admission is into the |
| 48 | // return generation (the rollback id) the source's install built, after |
| 49 | // PreparedCheck, sealed when the return seal was built from the stopped |
| 50 | // database, its baseline the return generation's pin at the return |
| 51 | // seal's maximum (genselect.SelectPrepared). A clean return's first is |
| 52 | // into a fresh generation restored from src_gen (the holder's generation |
| 53 | // ReturnToSource named), coverage clean-return; a declared or fenced |
| 54 | // return asks no coverage (Q6). A revert's first is into a fresh |
| 55 | // generation restored from the holder's own generation, which a revert |
| 56 | // never changed, with no coverage. A restart is a fresh generation with |
| 57 | // no coverage, restored from the certified remote (genselect.Select), or, |
| 58 | // for a dirty return while the location's marker is not yet the return's, |
| 59 | // the return generation's pin (H14). The launch inputs are exactly the |
| 60 | // recorded ones (Rollback.RecoveryInputs or Move.RecoveryInputs; the |
| 61 | // start policy carries them), the live release and the launch |
| 62 | // environment's report are theirs, and CARLOS_MOVE_PROBE=1 is in the |
| 63 | // child's environment, as for the landing's probe. |
| 64 | // - AdmissionEvidence: the same decisions re-derived and compared against |
| 65 | // a baseline selected again. While a declared loss or a target fence is |
| 66 | // pending, the writer replaces RemoteMaxTXID with the frozen |
| 67 | // predecessor's maximum from its LossSources call (writer_loss.go), and |
| 68 | // AdditionalWritesLost with the other frozen sources: the loss contract |
| 69 | // (movesloss.go, Task 4) accepts exactly that substitution. |
| 70 | // - Materialize, before every start: the held render and the held release |
| 71 | // of the recorded inputs, each under its own claim selection at the |
| 72 | // admitted capture (Works: held-config, held-release; MovesClaimPolicy's |
| 73 | // recovery rows bind them to the recorded inputs), the launch |
| 74 | // environment's report again, then, for a dirty return, the return |
| 75 | // generation's verified marker under the admitted writer's fence, then |
| 76 | // C8's Establish under the start's work, then the §5.1 start check. |
| 77 | // - Publish, PublishFenced and PublicationEvidence work as the landing's: |
| 78 | // the dirty return's prepared marker before replication, a fresh |
| 79 | // generation's marker after its own first root, and this process's |
| 80 | // receipt read back. |
| 81 | // |
| 82 | // Production: the moves wake source (movesWakeSources, Plan 5 Task 9b) |
| 83 | // registers this owner set, with both switches on. C8 is Task 6's |
| 84 | // provider, the claim's (bindClaimMovesFiles); without one every recovery |
| 85 | // kind is refused, typed. |
| 86 | type movesRecoveryOwners struct{ m *movesProbeOwners } |
| 87 | |
| 88 | // recoveryKind reports whether o is a recovery the recovery owners serve. |
| 89 | func recoveryKind(o hostclaim.Observed) bool { |
| 90 | return !o.Absent && o.Claim != nil && claimRecoveryKind(o.Claim.Op.Kind) |
| 91 | } |
| 92 | |
| 93 | // recoveryProbe is what one recovery probe's checks read. |
| 94 | type recoveryProbe struct { |
| 95 | homeRegion string |
| 96 | kind string |
| 97 | // op is the move id; holdOp the op holding the row: the rollback id for |
| 98 | // a return, the move id for a revert. |
| 99 | op, holdOp string |
| 100 | holder *hostclaim.Holder |
| 101 | rec *provision.Record |
| 102 | // inputs are the recorded recovery inputs. |
| 103 | inputs *provision.MoveInputs |
| 104 | // first: the holder is still at the source's stamp S, so no recovery |
| 105 | // admission has happened yet. |
| 106 | first bool |
| 107 | // dirty: a dirty return. uncovered: a declared or fenced return (Q6). |
| 108 | dirty, uncovered bool |
| 109 | srcGen string |
| 110 | predecessor string |
| 111 | // cut is the cut's journalled session (op.journal.cut_session). |
| 112 | cut string |
| 113 | // A dirty return's: the rollback id (its generation), the return seal |
| 114 | // and the install manifest its install journalled. |
| 115 | rb string |
| 116 | seal moves.Seal |
| 117 | manifest moves.InstallManifest |
| 118 | } |
| 119 | |
| 120 | // prepared is the dirty return's build as the landing's helpers read one: |
| 121 | // the return generation is the "op" whose verified marker is published. |
| 122 | func (v recoveryProbe) prepared() probeLanding { |
| 123 | return probeLanding{homeRegion: v.homeRegion, op: v.rb, holder: v.holder, rec: v.rec, seal: v.seal, manifest: v.manifest} |
| 124 | } |
| 125 | |
| 126 | // recoveryInputsChanged is a launch input that is not the recorded one: the |
| 127 | // next pass records the inputs again and probes again. |
| 128 | func recoveryInputsChanged(what string) error { |
| 129 | return fmt.Errorf("%w: the recovery's %s is not the recorded one", hostbox.ErrChanged, what) |
| 130 | } |
| 131 | |
| 132 | // recovery checks that o is a recovery probe this box may run and reads the |
| 133 | // record, the recorded inputs and, for a dirty return, the return seal and |
| 134 | // the journalled install manifest. |
| 135 | func (r movesRecoveryOwners) recovery(ctx context.Context, o hostclaim.Observed) (recoveryProbe, error) { |
| 136 | m := r.m |
| 137 | var v recoveryProbe |
| 138 | if !m.valid() { |
| 139 | return v, hostwrite.ErrUnavailable |
| 140 | } |
| 141 | if m.Files == nil { |
| 142 | return v, claimFilesRefusal() |
| 143 | } |
| 144 | if !recoveryKind(o) || o.Claim.Holder == nil { |
| 145 | return v, hostbox.ErrUnsupported |
| 146 | } |
| 147 | c := o.Claim |
| 148 | home, err := park.VerifiedHome(m.FleetHomeRegion, m.HomeRegion, m.Region, c.Op.Intent["home_region"]) |
| 149 | if err != nil { |
| 150 | return v, err |
| 151 | } |
| 152 | v.homeRegion, v.kind = home, c.Op.Kind |
| 153 | if c.Op.Phase != hostclaim.PhaseProbing || (v.kind == hostclaim.KindReverting && c.Op.Journal[hostclaim.JournalRevPhase] != hostclaim.PhaseProbing) { |
| 154 | return v, hostbox.ErrChanged |
| 155 | } |
| 156 | if c.Host != o.Host || c.Op.ID == "" || c.Op.Exec != m.Box || len(c.Op.Participants) == 0 || c.Op.Participants[0] != m.Box || |
| 157 | c.Op.Intent["source"] != m.Box || c.Op.Intent["target"] == m.Box || park.SameRegion(m.HomeRegion, c.Op.Intent["to"], m.Region) { |
| 158 | return v, hostbox.ErrChanged |
| 159 | } |
| 160 | for _, lost := range c.Lost { |
| 161 | if lost == m.Box { |
| 162 | return v, hostbox.ErrChanged |
| 163 | } |
| 164 | } |
| 165 | v.op, v.holder = c.Op.ID, c.Holder |
| 166 | stamp, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64) |
| 167 | srcGen, journalled := c.Op.Journal[hostclaim.JournalSrcGen] |
| 168 | if err != nil || stamp == 0 || !journalled { |
| 169 | return v, fmt.Errorf("%w: the source's stamp and generation are not journalled", hostbox.ErrChanged) |
| 170 | } |
| 171 | v.first, v.srcGen, v.cut = v.holder.Stamp == stamp, srcGen, c.Op.Journal[hostclaim.JournalCutSession] |
| 172 | rec, err := m.Records(ctx, v.holder.Account, v.holder.App, o.Host) |
| 173 | if err != nil { |
| 174 | return v, err |
| 175 | } |
| 176 | if rec == nil || rec.Host != o.Host || rec.Move == nil { |
| 177 | return v, hostbox.ErrChanged |
| 178 | } |
| 179 | mv := rec.Move |
| 180 | if mv.ID != v.op || mv.Incarnation != v.holder.Incarnation || !park.PlacementMatches(m.HomeRegion, mv.From, m.Region) { |
| 181 | return v, hostbox.ErrChanged |
| 182 | } |
| 183 | switch v.kind { |
| 184 | case hostclaim.KindRestoring: |
| 185 | rb := mv.Rollback |
| 186 | if mv.State != provision.MoveRollingBack || rb == nil || rb.ID == "" || rb.ID != c.Op.Intent["rollback"] || rb.SourcePhase != provision.RbSourceProbing { |
| 187 | return v, hostbox.ErrChanged |
| 188 | } |
| 189 | v.rb, v.holdOp, v.dirty, v.inputs = rb.ID, rb.ID, rb.Dirty, rb.RecoveryInputs |
| 190 | // A declared or fenced return (Q6); the record and the claim |
| 191 | // disagreeing on the target's loss wait (movesloss.go). |
| 192 | if v.uncovered, err = recoveryDeclared(c, rb.LossRef); err != nil { |
| 193 | return v, err |
| 194 | } |
| 195 | if v.dirty && v.uncovered { |
| 196 | return v, fmt.Errorf("%w: a declared or fenced return is never dirty", hostbox.ErrChanged) |
| 197 | } |
| 198 | default: // hostclaim.KindReverting |
| 199 | if mv.State != provision.MoveReverting || mv.ReopenedAt != nil { |
| 200 | return v, hostbox.ErrChanged |
| 201 | } |
| 202 | v.holdOp, v.inputs = v.op, mv.RecoveryInputs |
| 203 | } |
| 204 | if v.first && !v.dirty && v.holder.Generation != v.srcGen { |
| 205 | // A clean return names src_gen again (ReturnToSource); a revert |
| 206 | // never left it. |
| 207 | return v, fmt.Errorf("%w: the holder is not at the source's generation", hostbox.ErrChanged) |
| 208 | } |
| 209 | if v.inputs == nil || v.inputs.Release == "" { |
| 210 | return v, fmt.Errorf("%w: no recorded recovery inputs", hostbox.ErrChanged) |
| 211 | } |
| 212 | predecessor, err := predecessorRegion(v.holder, mv.From) |
| 213 | if err != nil { |
| 214 | return v, err |
| 215 | } |
| 216 | v.predecessor, v.rec = predecessor, rec |
| 217 | if !v.dirty { |
| 218 | return v, nil |
| 219 | } |
| 220 | seal, err := returnSeal(c) |
| 221 | if err != nil { |
| 222 | return v, err |
| 223 | } |
| 224 | v.seal = seal |
| 225 | j, err := moves.ReadJournal(m.DataDir, o.Host, v.rb) |
| 226 | if err != nil { |
| 227 | return v, err |
| 228 | } |
| 229 | raw, ok := j.Steps[landStepInstall] |
| 230 | if !ok { |
| 231 | return v, fmt.Errorf("%w: the return journalled no install manifest", hostbox.ErrChanged) |
| 232 | } |
| 233 | if err := json.Unmarshal([]byte(raw), &v.manifest); err != nil { |
| 234 | return v, fmt.Errorf("the journalled install manifest: %w", err) |
| 235 | } |
| 236 | if v.manifest.Op != v.rb || v.manifest.Host != o.Host || v.manifest.N != v.seal.N || v.manifest.N == 0 || v.manifest.SnapshotKey == "" || v.manifest.SnapshotVersion == "" { |
| 237 | return v, fmt.Errorf("%w: the install manifest is not this return's build at the seal", hostbox.ErrChanged) |
| 238 | } |
| 239 | return v, nil |
| 240 | } |
| 241 | |
| 242 | // recoveryLossPending reports a declared loss or a target fence the writer |
| 243 | // is about to admit (writer.go's routing): an unconsumed operation loss, or |
| 244 | // the consumed fence's frozen target writer before its first admission. |
| 245 | func recoveryLossPending(c *hostclaim.Claim) (bool, error) { |
| 246 | if _, pending := hostclaim.PendingMoveFenceLoss(c); pending { |
| 247 | return true, nil |
| 248 | } |
| 249 | id, err := hostclaim.OperationLossID(c) |
| 250 | if err != nil { |
| 251 | return false, err |
| 252 | } |
| 253 | return id != "" && !hostclaim.OperationLossConsumed(c, id), nil |
| 254 | } |
| 255 | |
| 256 | // preparedAdmission: a dirty return's first admission, into the return |
| 257 | // generation. |
| 258 | func (v recoveryProbe) preparedAdmission() bool { return v.dirty && v.first } |
| 259 | |
| 260 | // freshGeneration: g may name a recovery's fresh generation. |
| 261 | func (v recoveryProbe) freshGeneration(g string) bool { |
| 262 | return g != v.op && (v.rb == "" || g != v.rb) && claimFreshGeneration(v.holder, g) |
| 263 | } |
| 264 | |
| 265 | // admission is the recovery's admission and baseline for v, the new |
| 266 | // generation fresh unless it is the dirty return's first (see the file |
| 267 | // comment). dbPath is the held row's database, for PreparedCheck. |
| 268 | func (r movesRecoveryOwners) admission(ctx context.Context, o hostclaim.Observed, v recoveryProbe, dbPath, fresh string) (hostclaim.Admission, *genselect.Plan, error) { |
| 269 | m := r.m |
| 270 | h := v.holder |
| 271 | a := hostclaim.Admission{ObservedOpID: v.op, ObservedPhase: o.Claim.Op.Phase, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: m.Box, Mode: "cold", |
| 272 | Region: m.Region, PredecessorRegion: v.predecessor} |
| 273 | var ( |
| 274 | plan *genselect.Plan |
| 275 | err error |
| 276 | ) |
| 277 | if v.preparedAdmission() { |
| 278 | files, db, err := m.localMarkers(o.Host, dbPath) |
| 279 | if err != nil { |
| 280 | return a, nil, err |
| 281 | } |
| 282 | if err := moves.PreparedCheck(v.rb, *h, files, db); err != nil { |
| 283 | return a, nil, err |
| 284 | } |
| 285 | if !claimFreshGeneration(h, v.rb) { |
| 286 | return a, nil, hostbox.ErrChanged |
| 287 | } |
| 288 | a.NewGeneration = v.rb |
| 289 | if v.seal.FromDB { |
| 290 | a.Coverage = recoveryCoverageSealed |
| 291 | } |
| 292 | if plan, err = genselect.SelectPrepared(ctx, m.Data, m.request(o.Host, h), preparedPin(v.prepared())); err != nil { |
| 293 | return a, nil, err |
| 294 | } |
| 295 | // The maximum is the return seal's. |
| 296 | if plan.MaxTXID != v.seal.N || plan.Pin == nil || *plan.Pin != manifestPin(v.manifest) { |
| 297 | return a, nil, fmt.Errorf("%w: the return generation is not at the return seal", hostbox.ErrChanged) |
| 298 | } |
| 299 | } else { |
| 300 | a.NewGeneration = fresh |
| 301 | if v.first && v.kind == hostclaim.KindRestoring && !v.uncovered { |
| 302 | a.Coverage = recoveryCoverageClean |
| 303 | } |
| 304 | if plan, err = r.baseline(ctx, o.Host, v); err != nil { |
| 305 | return a, nil, err |
| 306 | } |
| 307 | } |
| 308 | a.RemoteMaxTXID = plan.MaxTXID |
| 309 | return a, plan, nil |
| 310 | } |
| 311 | |
| 312 | // baseline is a fresh generation's baseline: the certified remote; for a |
| 313 | // dirty return's restart while the location's marker is not yet the return |
| 314 | // generation's (nor a generation descending from it on this box), the |
| 315 | // return generation's pin (H14); for a clean return while the recovery has |
| 316 | // no root, the recorded source baseline (sourceBaseline, fix round 1 ruling |
| 317 | // 4). |
| 318 | func (r movesRecoveryOwners) baseline(ctx context.Context, host string, v recoveryProbe) (*genselect.Plan, error) { |
| 319 | m := r.m |
| 320 | if v.kind == hostclaim.KindRestoring && !v.dirty { |
| 321 | rooted, err := r.recoveryRooted(ctx, host, v) |
| 322 | if err != nil { |
| 323 | return nil, err |
| 324 | } |
| 325 | if !rooted { |
| 326 | return r.sourceBaseline(ctx, host, v) |
| 327 | } |
| 328 | } |
| 329 | if v.dirty { |
| 330 | digest, err := manifestDigest(v.manifest) |
| 331 | if err != nil { |
| 332 | return nil, err |
| 333 | } |
| 334 | marker, err := m.readMarker(ctx, v.holder) |
| 335 | if err != nil { |
| 336 | return nil, err |
| 337 | } |
| 338 | if !m.landingMarker(marker, host, v.prepared(), digest) { |
| 339 | return genselect.SelectPrepared(ctx, m.Data, m.request(host, v.holder), preparedPin(v.prepared())) |
| 340 | } |
| 341 | } |
| 342 | return genselect.Select(ctx, m.Data, m.request(host, v.holder)) |
| 343 | } |
| 344 | |
| 345 | // Select chooses the recovery probe's launch inputs and admission. It is |
| 346 | // read-only. |
| 347 | func (r movesRecoveryOwners) Select(ctx context.Context, rt registry.Route, o hostclaim.Observed, p activator.StartPolicy) (ClaimLaunchInputs, error) { |
| 348 | m := r.m |
| 349 | v, err := r.recovery(ctx, o) |
| 350 | if err != nil { |
| 351 | return ClaimLaunchInputs{}, err |
| 352 | } |
| 353 | if !p.Probe { |
| 354 | return ClaimLaunchInputs{}, hostbox.ErrUnsupported |
| 355 | } |
| 356 | if p.Inputs == nil || !sameInputs(v.inputs, *p.Inputs) || (p.Prepared != nil && (!v.dirty || *p.Prepared != v.manifest)) { |
| 357 | return ClaimLaunchInputs{}, hostbox.ErrChanged |
| 358 | } |
| 359 | if err := heldRoute(rt, o.Host, v.holdOp); err != nil { |
| 360 | return ClaimLaunchInputs{}, err |
| 361 | } |
| 362 | fresh := "" |
| 363 | if !v.preparedAdmission() { |
| 364 | if fresh = m.NewID(); !v.freshGeneration(fresh) { |
| 365 | return ClaimLaunchInputs{}, hostwrite.ErrUnavailable |
| 366 | } |
| 367 | } |
| 368 | admission, plan, err := r.admission(ctx, o, v, rt.DBPath, fresh) |
| 369 | if err != nil { |
| 370 | return ClaimLaunchInputs{}, err |
| 371 | } |
| 372 | h := v.holder |
| 373 | ready, err := m.ReadyPath(ctx, h.Account, h.App) |
| 374 | if err != nil { |
| 375 | return ClaimLaunchInputs{}, err |
| 376 | } |
| 377 | l, _, err := m.Launch(ctx, rt) |
| 378 | if err != nil { |
| 379 | return ClaimLaunchInputs{}, err |
| 380 | } |
| 381 | if !filepath.IsAbs(l.Executable) || !claimHexDigest(l.ExecutableSHA256) || l.Version == "" { |
| 382 | return ClaimLaunchInputs{}, hostwrite.ErrUnavailable |
| 383 | } |
| 384 | if l.Version != v.inputs.Release { |
| 385 | return ClaimLaunchInputs{}, recoveryInputsChanged("live release") |
| 386 | } |
| 387 | if v.inputs.EnvReport == "" || localEnvDigest(m.EnvKey, l.Env) != v.inputs.EnvReport { |
| 388 | return ClaimLaunchInputs{}, recoveryInputsChanged("launch environment") |
| 389 | } |
| 390 | return ClaimLaunchInputs{Roles: []string{"main"}, Identity: launchIdentity(v.inputs), Baseline: plan, Admission: admission, |
| 391 | Executable: l.Executable, ExecutableSHA256: l.ExecutableSHA256, Args: append([]string(nil), l.Args...), Env: probeEnv(l.Env), |
| 392 | DBPath: rt.DBPath, Version: l.Version, Limit: l.Limit, Prepared: v.preparedAdmission(), ReadyPath: ready}, nil |
| 393 | } |
| 394 | |
| 395 | // AdmissionEvidence re-derives Select's admission against the observation |
| 396 | // and a baseline selected again, immediately before the admission's CAS. |
| 397 | func (r movesRecoveryOwners) AdmissionEvidence(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, baseline *genselect.Plan) error { |
| 398 | m := r.m |
| 399 | v, err := r.recovery(ctx, o) |
| 400 | if err != nil { |
| 401 | return err |
| 402 | } |
| 403 | if baseline == nil || in.Mode != "cold" || in.PredecessorCertified || in.LossID != "" || in.KeepStamp { |
| 404 | return hostbox.ErrChanged |
| 405 | } |
| 406 | // Under a pending loss or fence the writer substitutes the validated |
| 407 | // sources' maxima; only the loss contract vouches for them |
| 408 | // (movesloss.go). Otherwise the admission is Select's. |
| 409 | pending, err := recoveryLossPending(o.Claim) |
| 410 | if err != nil { |
| 411 | return err |
| 412 | } |
| 413 | if pending { |
| 414 | if err := r.lossEvidence(ctx, o, in, baseline); err != nil { |
| 415 | return err |
| 416 | } |
| 417 | } else if len(in.AdditionalWritesLost) != 0 || in.RemoteMaxTXID != baseline.MaxTXID { |
| 418 | return hostbox.ErrChanged |
| 419 | } |
| 420 | if !v.preparedAdmission() && !v.freshGeneration(in.NewGeneration) { |
| 421 | return hostbox.ErrChanged |
| 422 | } |
| 423 | rt, err := m.route(o.Host, v.holdOp) |
| 424 | if err != nil { |
| 425 | return err |
| 426 | } |
| 427 | want, fresh, err := r.admission(ctx, o, v, rt.DBPath, in.NewGeneration) |
| 428 | if err != nil { |
| 429 | return err |
| 430 | } |
| 431 | if in.ObservedOpID != want.ObservedOpID || in.ObservedPhase != want.ObservedPhase || in.ObservedSession != want.ObservedSession || in.ObservedGeneration != want.ObservedGeneration || |
| 432 | in.Box != want.Box || !park.SameRegion(m.HomeRegion, in.Region, m.Region) || !park.SameRegion(v.homeRegion, in.PredecessorRegion, want.PredecessorRegion) || |
| 433 | in.NewGeneration != want.NewGeneration || in.Coverage != want.Coverage { |
| 434 | return hostbox.ErrChanged |
| 435 | } |
| 436 | if fresh.Generation != baseline.Generation || fresh.MaxTXID != baseline.MaxTXID || !reflect.DeepEqual(fresh.Files, baseline.Files) || !reflect.DeepEqual(fresh.Pin, baseline.Pin) { |
| 437 | return hostbox.ErrChanged |
| 438 | } |
| 439 | // A clean return's source baseline is pinned here, the owner's last |
| 440 | // call before the admission's CAS and so before the activator restores |
| 441 | // it (claimwake.go Restore precedes Materialize): a crash after the |
| 442 | // restore leaves the pin a restart must match (fix round 2). |
| 443 | // So is this admission's link in the recovery's lineage: the |
| 444 | // generation it admits and the one it observed (fix round 3). |
| 445 | if err := r.recordLineage(o.Host, v, in.NewGeneration); err != nil { |
| 446 | return err |
| 447 | } |
| 448 | return r.recordSourceBaseline(o.Host, v, baseline) |
| 449 | } |
| 450 | |
| 451 | // Materialize runs before every start of the admitted recovery probe, in the |
| 452 | // ruled order (see the file comment). The spawn follows only if all of it |
| 453 | // passed. |
| 454 | func (r movesRecoveryOwners) Materialize(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { |
| 455 | m := r.m |
| 456 | v, err := r.recovery(ctx, o) |
| 457 | if err != nil { |
| 458 | return err |
| 459 | } |
| 460 | if err := admittedProbe(o, in, m.Box); err != nil { |
| 461 | return err |
| 462 | } |
| 463 | if work == nil { |
| 464 | return hostwrite.ErrUnavailable |
| 465 | } |
| 466 | if in.Identity != launchIdentity(v.inputs) { |
| 467 | return hostbox.ErrChanged |
| 468 | } |
| 469 | fence := writerFence(work, in.Identity) |
| 470 | if err := fence(ctx); err != nil { |
| 471 | return err |
| 472 | } |
| 473 | rt, err := m.route(o.Host, v.holdOp) |
| 474 | if err != nil { |
| 475 | return err |
| 476 | } |
| 477 | if rt.DBPath != in.DBPath { |
| 478 | return hostbox.ErrChanged |
| 479 | } |
| 480 | h := v.holder |
| 481 | captured := hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp} |
| 482 | hr, err := m.Render(ctx, rt, v.holdOp, func(pre appconfig.HeldRender) (hostwrite.Work, error) { |
| 483 | // The recorded inputs decide before any selection or write. |
| 484 | if !sameBundle(pre.BundleETags, v.inputs.BundleETags) || pre.OverlayETag != v.inputs.OverlayETag { |
| 485 | return nil, recoveryInputsChanged("configuration") |
| 486 | } |
| 487 | w, err := m.Works(ctx, o.Host, movesActionHeldConfig, captured) |
| 488 | if err != nil { |
| 489 | return nil, err |
| 490 | } |
| 491 | return heldIdentityWork{Work: w, ID: HeldConfigIdentity(pre.BundleETags, pre.OverlayETag)}, nil |
| 492 | }) |
| 493 | if err != nil { |
| 494 | return err |
| 495 | } |
| 496 | if !sameBundle(hr.BundleETags, v.inputs.BundleETags) || hr.OverlayETag != v.inputs.OverlayETag { |
| 497 | return recoveryInputsChanged("configuration") |
| 498 | } |
| 499 | released, err := m.Works(ctx, o.Host, movesActionHeldRelease, captured) |
| 500 | if err != nil { |
| 501 | return err |
| 502 | } |
| 503 | if err := m.Release(ctx, rt, v.inputs.Release, heldIdentityWork{Work: released, ID: hostwrite.Identity{Release: v.inputs.Release}}); err != nil { |
| 504 | return err |
| 505 | } |
| 506 | if localEnvDigest(m.EnvKey, in.Env) != v.inputs.EnvReport { |
| 507 | return recoveryInputsChanged("launch environment") |
| 508 | } |
| 509 | m.recordFence(o.Host, h.Session, work) |
| 510 | // The start gate: the tree is the one the recovery restored before its |
| 511 | // first authorized start, and the recovery's own after it. |
| 512 | if err := r.startGate(ctx, o, v, work, in.Identity); err != nil { |
| 513 | return err |
| 514 | } |
| 515 | if v.dirty { |
| 516 | receipt, err := m.preparedMarker(ctx, o.Host, v.prepared(), fence) |
| 517 | if err != nil { |
| 518 | return err |
| 519 | } |
| 520 | if in.Prepared { |
| 521 | m.record(o.Host, h.Session, receipt) |
| 522 | } |
| 523 | } |
| 524 | if err := m.Files.Establish(ctx, o, in, work); err != nil { |
| 525 | return err |
| 526 | } |
| 527 | return m.startCheck(ctx, o, in) |
| 528 | } |
| 529 | |
| 530 | // EmptyAuthority: a recovery never admits the empty bootstrap. |
| 531 | func (r movesRecoveryOwners) EmptyAuthority(context.Context, hostclaim.Observed, ClaimLaunchInputs) error { |
| 532 | return hostbox.ErrUnsupported |
| 533 | } |
| 534 | |
| 535 | // Publish is Group D's call for the dirty return's prepared admission, |
| 536 | // before replication: Materialize published the return generation's |
| 537 | // verified marker; this finds it under the admitted writer's fence and |
| 538 | // records the receipt. |
| 539 | func (r movesRecoveryOwners) Publish(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { |
| 540 | m := r.m |
| 541 | v, err := r.recovery(ctx, o) |
| 542 | if err != nil { |
| 543 | return err |
| 544 | } |
| 545 | if err := admittedProbe(o, in, m.Box); err != nil { |
| 546 | return err |
| 547 | } |
| 548 | h := v.holder |
| 549 | if !v.dirty || !in.Prepared || in.Admission.NewGeneration != v.rb || h.Generation != v.rb { |
| 550 | return hostbox.ErrChanged |
| 551 | } |
| 552 | work := m.fence(o.Host, h.Session) |
| 553 | if work == nil { |
| 554 | return fmt.Errorf("%w: no admitted writer's fence for session %d", hostbox.ErrChanged, h.Session) |
| 555 | } |
| 556 | receipt, err := m.preparedMarker(ctx, o.Host, v.prepared(), writerFence(work, in.Identity)) |
| 557 | if err != nil { |
| 558 | return err |
| 559 | } |
| 560 | if receipt.MarkerVersion == "" { |
| 561 | return hostbox.ErrChanged |
| 562 | } |
| 563 | m.record(o.Host, h.Session, receipt) |
| 564 | return nil |
| 565 | } |
| 566 | |
| 567 | // PublishFenced writes a fresh recovery generation's marker after the |
| 568 | // session's own first root, under the publisher's fence, as the landing's |
| 569 | // restarted attempt and serving do. |
| 570 | func (r movesRecoveryOwners) PublishFenced(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, pub ClaimPublication) (ClaimPublicationReceipt, error) { |
| 571 | m := r.m |
| 572 | v, err := r.recovery(ctx, o) |
| 573 | if err != nil { |
| 574 | return ClaimPublicationReceipt{}, err |
| 575 | } |
| 576 | if err := admittedProbe(o, in, m.Box); err != nil { |
| 577 | return ClaimPublicationReceipt{}, err |
| 578 | } |
| 579 | if pub.Sync == nil || pub.Check == nil { |
| 580 | return ClaimPublicationReceipt{}, hostwrite.ErrUnavailable |
| 581 | } |
| 582 | h := v.holder |
| 583 | if in.Prepared || h.Generation == v.op || (v.rb != "" && h.Generation == v.rb) { |
| 584 | return ClaimPublicationReceipt{}, hostbox.ErrChanged |
| 585 | } |
| 586 | work := m.fence(o.Host, h.Session) |
| 587 | if work == nil { |
| 588 | return ClaimPublicationReceipt{}, fmt.Errorf("%w: no admitted writer's fence for session %d", hostbox.ErrChanged, h.Session) |
| 589 | } |
| 590 | // The files on disk must be the tree the recovery placed, held by this |
| 591 | // recovery's admitted lineage, before anything is published (fix round |
| 592 | // 3): its own writes since its first start are authoritative. |
| 593 | placed, ok, err := r.placement(o.Host, v) |
| 594 | if err != nil { |
| 595 | return ClaimPublicationReceipt{}, err |
| 596 | } |
| 597 | if !ok { |
| 598 | return ClaimPublicationReceipt{}, fmt.Errorf("%w: the recovery recorded no placed tree", hostbox.ErrChanged) |
| 599 | } |
| 600 | if err := work.With(ctx, in.Identity, func() error { |
| 601 | _, err := r.vouchFiles(o.Host, v, placed) |
| 602 | return err |
| 603 | }); err != nil { |
| 604 | return ClaimPublicationReceipt{}, err |
| 605 | } |
| 606 | return m.publishFresh(ctx, o.Host, h, pub, func() error { return r.refreshFilesGen(ctx, o.Host, v, placed, work, in.Identity) }) |
| 607 | } |
| 608 | |
| 609 | // vouchFiles requires the files on disk to be the placed tree (identity: a |
| 610 | // replaced tree is refused) and the holder's session to be this recovery's |
| 611 | // own through its admitted lineage, and reports whether .files-gen will |
| 612 | // move to holder.generation: it names that generation's cold predecessor |
| 613 | // for this incarnation. It never checks content: since the first start the |
| 614 | // tree is the recovery's, and its writes are authoritative (fix round 3, |
| 615 | // spec §7.8). The caller runs it under the session's fence. |
| 616 | func (r movesRecoveryOwners) vouchFiles(host string, v recoveryProbe, placed recoveryPlacement) (bool, error) { |
| 617 | m := r.m |
| 618 | h := v.holder |
| 619 | if err := r.verifyPlaced(host, placed); err != nil { |
| 620 | return false, err |
| 621 | } |
| 622 | if err := r.lineage(host, v); err != nil { |
| 623 | return false, err |
| 624 | } |
| 625 | dir, _ := m.FilesDir(host) |
| 626 | files, _, err := m.localMarkers(host, filepath.Join(dir, ".none")) |
| 627 | if err != nil || files == nil || files.Incarnation != h.Incarnation || files.Op == h.Generation { |
| 628 | return false, err |
| 629 | } |
| 630 | if pred, ok := coldPredecessor(h, h.Generation); !ok || pred == "" || files.Op != pred { |
| 631 | return false, nil |
| 632 | } |
| 633 | return true, nil |
| 634 | } |
| 635 | |
| 636 | // refreshFilesGen points .files-gen at the published generation, inside the |
| 637 | // admitted writer's fence, after the files on disk are proved the tree the |
| 638 | // recovery placed and the session this recovery's own (vouchFiles); a |
| 639 | // replaced tree or an unproven session fails the publication and relabels |
| 640 | // nothing. It relabels only a .files-gen naming the generation's cold |
| 641 | // predecessor for this incarnation (the files the start check accepted for |
| 642 | // this session); absent, or anything else, is left for the start check to |
| 643 | // decide. |
| 644 | func (r movesRecoveryOwners) refreshFilesGen(ctx context.Context, host string, v recoveryProbe, placed recoveryPlacement, work hostwrite.Work, id hostwrite.Identity) error { |
| 645 | m := r.m |
| 646 | h := v.holder |
| 647 | dir, _ := m.FilesDir(host) |
| 648 | return work.With(ctx, id, func() error { |
| 649 | relabel, err := r.vouchFiles(host, v, placed) |
| 650 | if err != nil || !relabel { |
| 651 | return err |
| 652 | } |
| 653 | body, err := json.Marshal(moves.FilesGen{Op: h.Generation, Incarnation: h.Incarnation}) |
| 654 | if err != nil { |
| 655 | return err |
| 656 | } |
| 657 | if err := syncPath(dir); err != nil { |
| 658 | return err |
| 659 | } |
| 660 | return writeFileAtomic(filepath.Join(dir, archive.MarkerName), body, 0o644) |
| 661 | }) |
| 662 | } |
| 663 | |
| 664 | // PublicationEvidence requires this process's receipt for the admitted |
| 665 | // session, as the landing's does; the return generation's marker also |
| 666 | // carries its build's pin and install.json's digest. |
| 667 | func (r movesRecoveryOwners) PublicationEvidence(ctx context.Context, o hostclaim.Observed) error { |
| 668 | v, err := r.recovery(ctx, o) |
| 669 | if err != nil { |
| 670 | return err |
| 671 | } |
| 672 | prepared := "" |
| 673 | if v.dirty { |
| 674 | prepared = v.rb |
| 675 | } |
| 676 | return r.m.publicationEvidence(ctx, o.Host, v.holder, prepared, v.manifest) |
| 677 | } |
| 678 | |
| 679 | // rcStepFilesPlaced is the recovery journal's record of the tree it placed |
| 680 | // (JSON recoveryPlacement): the files directory's and <host>.env's |
| 681 | // identities and the tree's plain digest before the recovery's first start. |
| 682 | // The recovery journal is the rollback's (a return) or the move's (a |
| 683 | // revert). |
| 684 | const rcStepFilesPlaced = "rc_files_placed" |
| 685 | |
| 686 | // rcStepLineage prefixes the recovery journal's record of each generation |
| 687 | // the recovery admitted (rcStepLineage+<generation>, JSON recoveryLineage). |
| 688 | const rcStepLineage = "rc_lineage." |
| 689 | |
| 690 | // recoveryPlacement is what rcStepFilesPlaced records. |
| 691 | type recoveryPlacement struct { |
| 692 | Files string `json:"files"` |
| 693 | Env string `json:"env,omitempty"` |
| 694 | Plain string `json:"plain"` |
| 695 | } |
| 696 | |
| 697 | // journalOp is the recovery's journal: the rollback's for a return, the |
| 698 | // move's for a revert. |
| 699 | func (v recoveryProbe) journalOp() string { |
| 700 | if v.kind == hostclaim.KindRestoring { |
| 701 | return v.rb |
| 702 | } |
| 703 | return v.op |
| 704 | } |
| 705 | |
| 706 | // placement reads the recorded placed tree; false when none is recorded. |
| 707 | func (r movesRecoveryOwners) placement(host string, v recoveryProbe) (recoveryPlacement, bool, error) { |
| 708 | j, err := moves.ReadJournal(r.m.DataDir, host, v.journalOp()) |
| 709 | if err != nil { |
| 710 | return recoveryPlacement{}, false, err |
| 711 | } |
| 712 | raw, ok := j.Steps[rcStepFilesPlaced] |
| 713 | if !ok { |
| 714 | return recoveryPlacement{}, false, nil |
| 715 | } |
| 716 | var p recoveryPlacement |
| 717 | if err := json.Unmarshal([]byte(raw), &p); err != nil || p.Files == "" || p.Plain == "" { |
| 718 | return recoveryPlacement{}, false, fmt.Errorf("%w: the recorded placed tree does not read (%v)", hostbox.ErrChanged, err) |
| 719 | } |
| 720 | return p, true, nil |
| 721 | } |
| 722 | |
| 723 | // startGate runs under the start's work before every start. Before the |
| 724 | // recovery's first authorized start (no placement journalled) the tree must |
| 725 | // be the one the recovery restored (recordPlacement); after it, the tree is |
| 726 | // the recovery's own: its identity is the placed one, and its content is |
| 727 | // never compared again (fix round 3). Every start's session must be this |
| 728 | // recovery's through its admitted lineage. |
| 729 | func (r movesRecoveryOwners) startGate(ctx context.Context, o hostclaim.Observed, v recoveryProbe, work hostwrite.Work, id hostwrite.Identity) error { |
| 730 | placed, ok, err := r.placement(o.Host, v) |
| 731 | if err != nil { |
| 732 | return err |
| 733 | } |
| 734 | if !ok { |
| 735 | expected, err := restoredDigest(o.Claim, v) |
| 736 | if err != nil { |
| 737 | return err |
| 738 | } |
| 739 | if err := r.recordPlacement(ctx, o.Host, v, expected, work, id); err != nil { |
| 740 | return err |
| 741 | } |
| 742 | } else if err := work.With(ctx, id, func() error { return r.verifyPlaced(o.Host, placed) }); err != nil { |
| 743 | return err |
| 744 | } |
| 745 | return r.lineage(o.Host, v) |
| 746 | } |
| 747 | |
| 748 | // restoredDigest is the plain digest of the tree the recovery restores, as |
| 749 | // its baselines record it: a dirty return's is the return baseline the |
| 750 | // target journalled in the claim (the tree its install extracted); a clean |
| 751 | // return's or a revert's is the source's stopped baseline (Move.Source, |
| 752 | // which the clean verification and the revert's verify compared). "" when |
| 753 | // the record keeps none (a revert from stopping, J8): the first start then |
| 754 | // takes the tree as it finds it. |
| 755 | func restoredDigest(c *hostclaim.Claim, v recoveryProbe) (string, error) { |
| 756 | if v.dirty { |
| 757 | raw, ok := c.Op.Journal[moves.JournalReturnBaseline] |
| 758 | if !ok { |
| 759 | return "", fmt.Errorf("%w: the claim carries no return baseline", hostbox.ErrChanged) |
| 760 | } |
| 761 | b, err := moves.DecodeReturnBaseline(raw) |
| 762 | if err != nil { |
| 763 | return "", err |
| 764 | } |
| 765 | return b.Files.Plain, nil |
| 766 | } |
| 767 | if src := v.rec.Move.Source; src != nil && src.Files != nil { |
| 768 | return src.Files.Plain, nil |
| 769 | } |
| 770 | return "", nil |
| 771 | } |
| 772 | |
| 773 | // recordPlacement journals the tree the recovery restored, as placed, under |
| 774 | // the start's work, before the recovery's first start: the files directory's |
| 775 | // and <host>.env's identities and the plain digest, which must be expected |
| 776 | // (the restored tree's, restoredDigest) when that is known. A dirty |
| 777 | // return's tree is the one its install placed (landStepFilesPlaced), and |
| 778 | // must still be. A tree that is not the restored one is refused and nothing |
| 779 | // is journalled, so the next attempt checks again. Once recorded it is never |
| 780 | // rewritten. |
| 781 | func (r movesRecoveryOwners) recordPlacement(ctx context.Context, host string, v recoveryProbe, expected string, work hostwrite.Work, id hostwrite.Identity) error { |
| 782 | m := r.m |
| 783 | dir, env := m.FilesDir(host) |
| 784 | var p recoveryPlacement |
| 785 | if err := work.With(ctx, id, func() error { |
| 786 | var err error |
| 787 | if p.Files, err = pathIdentity(dir); err != nil { |
| 788 | return err |
| 789 | } |
| 790 | switch envID, err := pathIdentity(env); { |
| 791 | case err == nil: |
| 792 | p.Env = envID |
| 793 | case !errors.Is(err, fs.ErrNotExist): |
| 794 | return err |
| 795 | } |
| 796 | if v.dirty { |
| 797 | j, err := moves.ReadJournal(m.DataDir, host, v.rb) |
| 798 | if err != nil { |
| 799 | return err |
| 800 | } |
| 801 | installed, err := journalledStaged(j, landStepFilesPlaced) |
| 802 | if err != nil { |
| 803 | return fmt.Errorf("%w: %v", hostbox.ErrChanged, err) |
| 804 | } |
| 805 | if installed.Files != p.Files || installed.Env != p.Env { |
| 806 | return fmt.Errorf("%w: the files are not the tree the return installed", hostbox.ErrChanged) |
| 807 | } |
| 808 | } |
| 809 | if p.Plain, err = m.filesDigest(host); err != nil { |
| 810 | return err |
| 811 | } |
| 812 | if expected != "" && p.Plain != expected { |
| 813 | return fmt.Errorf("%w: the files are not the tree the recovery restored", hostbox.ErrChanged) |
| 814 | } |
| 815 | return nil |
| 816 | }); err != nil { |
| 817 | return err |
| 818 | } |
| 819 | body, err := json.Marshal(p) |
| 820 | if err != nil { |
| 821 | return err |
| 822 | } |
| 823 | m.mu.Lock() |
| 824 | defer m.mu.Unlock() |
| 825 | err = moves.UpdateJournal(m.DataDir, host, v.journalOp(), func(j *moves.Journal) error { |
| 826 | if _, ok := j.Steps[rcStepFilesPlaced]; ok { |
| 827 | return errJournalUnchanged |
| 828 | } |
| 829 | j.Mark(rcStepFilesPlaced, string(body)) |
| 830 | return nil |
| 831 | }) |
| 832 | if errors.Is(err, errJournalUnchanged) { |
| 833 | return nil |
| 834 | } |
| 835 | return err |
| 836 | } |
| 837 | |
| 838 | // verifyPlaced requires the files directory and <host>.env on disk to be |
| 839 | // the ones the recovery placed. |
| 840 | func (r movesRecoveryOwners) verifyPlaced(host string, p recoveryPlacement) error { |
| 841 | dir, env := r.m.FilesDir(host) |
| 842 | if ok, err := sameIdentity(dir, p.Files); err != nil || !ok { |
| 843 | return fmt.Errorf("%w: %s is not the files directory the recovery placed (%v)", hostbox.ErrChanged, dir, err) |
| 844 | } |
| 845 | if p.Env != "" { |
| 846 | if ok, err := sameIdentity(env, p.Env); err != nil || !ok { |
| 847 | return fmt.Errorf("%w: %s is not the environment file the recovery placed (%v)", hostbox.ErrChanged, env, err) |
| 848 | } |
| 849 | } |
| 850 | return nil |
| 851 | } |
| 852 | |
| 853 | // recoveryLineage is one rcStepLineage record: the generation an admission |
| 854 | // of this recovery observed (its predecessor), and whether it was the |
| 855 | // recovery's first admission (Root: the holder still at the source's stamp |
| 856 | // S). |
| 857 | type recoveryLineage struct { |
| 858 | Predecessor string `json:"predecessor"` |
| 859 | Root bool `json:"root,omitempty"` |
| 860 | } |
| 861 | |
| 862 | // recordLineage journals the admission's link before its CAS, in its |
| 863 | // evidence: the generation it admits, the generation it observed. The |
| 864 | // claim's CAS admits only over the observed generation, so a holder at a |
| 865 | // journalled generation proves that this recovery admitted it over its |
| 866 | // predecessor. A link for a generation never admitted (a refused CAS) is |
| 867 | // inert: generation names are fresh. |
| 868 | func (r movesRecoveryOwners) recordLineage(host string, v recoveryProbe, g string) error { |
| 869 | m := r.m |
| 870 | body, err := json.Marshal(recoveryLineage{Predecessor: v.holder.Generation, Root: v.first}) |
| 871 | if err != nil { |
| 872 | return err |
| 873 | } |
| 874 | m.mu.Lock() |
| 875 | defer m.mu.Unlock() |
| 876 | err = moves.UpdateJournal(m.DataDir, host, v.journalOp(), func(j *moves.Journal) error { |
| 877 | if have, ok := j.Steps[rcStepLineage+g]; ok { |
| 878 | if have != string(body) { |
| 879 | return fmt.Errorf("%w: generation %s is already another admission's", hostbox.ErrChanged, g) |
| 880 | } |
| 881 | return errJournalUnchanged |
| 882 | } |
| 883 | j.Mark(rcStepLineage+g, string(body)) |
| 884 | return nil |
| 885 | }) |
| 886 | if errors.Is(err, errJournalUnchanged) { |
| 887 | return nil |
| 888 | } |
| 889 | return err |
| 890 | } |
| 891 | |
| 892 | // lineage proves the holder's session is this recovery's own: this box |
| 893 | // writes the holder's generation, admitted in this region, and from it the |
| 894 | // journalled links reach the recovery's first admission, each generation |
| 895 | // admitted by this recovery over the one before. Any other writer admitted |
| 896 | // in between breaks the chain (its generation was never journalled here): |
| 897 | // the ownership interval is unproven and refused. |
| 898 | // |
| 899 | // The links bind generations, not sessions, and that is complete because |
| 900 | // the claim admits no writer during a restoring or reverting op except |
| 901 | // through these owners, and never a warm one (fix round 4; checked against |
| 902 | // the claim code at e0f4d1f7): |
| 903 | // |
| 904 | // - the holder's session, generation and writer box change only in |
| 905 | // AdmitWriter's CAS (hostclaim/generation.go:166-170); ReturnToSource |
| 906 | // sets the generation and stamp only from released to claimed, before |
| 907 | // any recovery admission (hostclaim/move.go:1025-1046); |
| 908 | // - AdmitWriter's only production callers are hostbox.Writer's three |
| 909 | // paths (hostbox/writer.go:63; writer_loss.go:58 through |
| 910 | // hostclaim/lost.go:218; writer_loss.go:79 through |
| 911 | // hostclaim/move_fence_admission.go:44); |
| 912 | // - Writer.Admit refuses every warm or certified admission before any |
| 913 | // path (hostbox/writer.go:34-36), so no admission keeps a generation |
| 914 | // while advancing the session: the counterexample's intervening warm |
| 915 | // admission cannot be written; |
| 916 | // - only the op's executor admits (hostbox/writer.go:37, |
| 917 | // hostclaim/generation.go:40-42), under the observed op and phase |
| 918 | // (hostclaim/generation.go:43-45); |
| 919 | // - every cold admission's evidence, on every path, is |
| 920 | // ClaimOwnerDispatch.AdmissionEvidence (host/claimassembly.go:181, |
| 921 | // host/claimconstruction.go:267), which hands restoring and reverting |
| 922 | // only to the moves owners (host/claimowners.go:220-224), which hand them |
| 923 | // to these recovery owners (movesprobeowners.go, recoveryKind); their |
| 924 | // AdmissionEvidence journals the link before it returns nil. Without |
| 925 | // the moves owners bound, the serving owners' evidence refuses any move |
| 926 | // kind (host/claimowners.go:328). |
| 927 | // |
| 928 | // So every session after the root is a cold admission evidenced here, and |
| 929 | // a chain of generations is a chain of this recovery's sessions. |
| 930 | // TestRecoveryOpAdmitsNoOutsideWriter pins the refusals. |
| 931 | func (r movesRecoveryOwners) lineage(host string, v recoveryProbe) error { |
| 932 | m := r.m |
| 933 | h := v.holder |
| 934 | n := len(h.Admissions) |
| 935 | if n == 0 || h.WriterBox != m.Box { |
| 936 | return fmt.Errorf("%w: this box does not write the holder's session", hostbox.ErrChanged) |
| 937 | } |
| 938 | if a := h.Admissions[n-1]; a.Box != m.Box || a.Generation != h.Generation || !park.SameRegion(m.HomeRegion, a.Region, m.Region) { |
| 939 | return fmt.Errorf("%w: the holder's admission is not this box's", hostbox.ErrChanged) |
| 940 | } |
| 941 | j, err := moves.ReadJournal(m.DataDir, host, v.journalOp()) |
| 942 | if err != nil { |
| 943 | return err |
| 944 | } |
| 945 | seen := map[string]bool{} |
| 946 | for g := h.Generation; !seen[g]; { |
| 947 | seen[g] = true |
| 948 | raw, ok := j.Steps[rcStepLineage+g] |
| 949 | if !ok { |
| 950 | return fmt.Errorf("%w: generation %s is not this recovery's admission", hostbox.ErrChanged, g) |
| 951 | } |
| 952 | var link recoveryLineage |
| 953 | if err := json.Unmarshal([]byte(raw), &link); err != nil { |
| 954 | return fmt.Errorf("%w: the lineage of %s does not read: %v", hostbox.ErrChanged, g, err) |
| 955 | } |
| 956 | if link.Root { |
| 957 | return nil |
| 958 | } |
| 959 | g = link.Predecessor |
| 960 | } |
| 961 | return fmt.Errorf("%w: the recovery's lineage loops", hostbox.ErrChanged) |
| 962 | } |
| 963 | |
| 964 | // rcStepSourceBaseline is the rollback journal's record of a clean return's |
| 965 | // source baseline (JSON genselect.Plan): src_gen's files by exact version, |
| 966 | // as the first admission that restores them was evidenced. |
| 967 | const rcStepSourceBaseline = "rc_source_baseline" |
| 968 | |
| 969 | // recoveryRooted reports whether the clean return's recovery has a root: an |
| 970 | // object under a generation one of its admissions made (this box, this |
| 971 | // region, a session after the cut's). Before it has, ordinary selection |
| 972 | // would follow the claim's predecessor chain through the target's admission |
| 973 | // (AdmitWriter records it as the first recovery admission's predecessor, |
| 974 | // though the restore was from src_gen) and can end in a covered baseline. |
| 975 | func (r movesRecoveryOwners) recoveryRooted(ctx context.Context, host string, v recoveryProbe) (bool, error) { |
| 976 | m := r.m |
| 977 | h := v.holder |
| 978 | cut, err := strconv.ParseUint(v.cut, 10, 64) |
| 979 | if err != nil { |
| 980 | return false, fmt.Errorf("%w: the cut's session is not journalled", hostbox.ErrChanged) |
| 981 | } |
| 982 | location := strings.TrimSuffix(h.Location, "/") |
| 983 | for _, a := range h.Admissions { |
| 984 | if a.Box != m.Box || a.Session <= cut || a.Generation == "" || !park.SameRegion(m.HomeRegion, a.Region, m.Region) { |
| 985 | continue |
| 986 | } |
| 987 | prefix := location + "/g-" + a.Generation + "/" |
| 988 | versions, err := m.Data.ListVersions(ctx, m.Bucket, prefix) |
| 989 | if err != nil { |
| 990 | return false, err |
| 991 | } |
| 992 | for _, ver := range versions { |
| 993 | if strings.HasPrefix(ver.Key, prefix) && ver.IsLatest && !ver.DeleteMarker { |
| 994 | return true, nil |
| 995 | } |
| 996 | } |
| 997 | } |
| 998 | return false, nil |
| 999 | } |
| 1000 | |
| 1001 | // sourceBaseline is a clean return's restore point before the recovery has |
| 1002 | // a root: src_gen in this region, selected as the first admission selected |
| 1003 | // it (the holder viewed at src_gen, whose marker names it), and, once |
| 1004 | // recorded, exactly the recorded files by version. |
| 1005 | func (r movesRecoveryOwners) sourceBaseline(ctx context.Context, host string, v recoveryProbe) (*genselect.Plan, error) { |
| 1006 | m := r.m |
| 1007 | view := *v.holder |
| 1008 | view.Generation = v.srcGen |
| 1009 | plan, err := genselect.Select(ctx, m.Data, m.request(host, &view)) |
| 1010 | if err != nil { |
| 1011 | return nil, err |
| 1012 | } |
| 1013 | if plan.Generation != selectedGeneration(v.srcGen) { |
| 1014 | return nil, fmt.Errorf("%w: the source baseline is not src_gen", hostbox.ErrChanged) |
| 1015 | } |
| 1016 | recorded, ok, err := r.recordedSourceBaseline(host, v) |
| 1017 | if err != nil { |
| 1018 | return nil, err |
| 1019 | } |
| 1020 | if ok && (recorded.Generation != plan.Generation || recorded.MaxTXID != plan.MaxTXID || !reflect.DeepEqual(recorded.Files, plan.Files) || !reflect.DeepEqual(recorded.Pin, plan.Pin)) { |
| 1021 | return nil, fmt.Errorf("%w: src_gen is not the recorded source baseline", hostbox.ErrChanged) |
| 1022 | } |
| 1023 | return plan, nil |
| 1024 | } |
| 1025 | |
| 1026 | func (r movesRecoveryOwners) recordedSourceBaseline(host string, v recoveryProbe) (genselect.Plan, bool, error) { |
| 1027 | j, err := moves.ReadJournal(r.m.DataDir, host, v.journalOp()) |
| 1028 | if err != nil { |
| 1029 | return genselect.Plan{}, false, err |
| 1030 | } |
| 1031 | raw, ok := j.Steps[rcStepSourceBaseline] |
| 1032 | if !ok { |
| 1033 | return genselect.Plan{}, false, nil |
| 1034 | } |
| 1035 | var p genselect.Plan |
| 1036 | if err := json.Unmarshal([]byte(raw), &p); err != nil || p.Generation == "" || len(p.Files) == 0 { |
| 1037 | return genselect.Plan{}, false, fmt.Errorf("%w: the recorded source baseline does not read (%v)", hostbox.ErrChanged, err) |
| 1038 | } |
| 1039 | return p, true, nil |
| 1040 | } |
| 1041 | |
| 1042 | // recordSourceBaseline journals a clean return's source baseline from the |
| 1043 | // admission's evidence, before the admission's CAS and the restore; once |
| 1044 | // recorded it is never rewritten. |
| 1045 | func (r movesRecoveryOwners) recordSourceBaseline(host string, v recoveryProbe, plan *genselect.Plan) error { |
| 1046 | if v.kind != hostclaim.KindRestoring || v.dirty || plan == nil || plan.Generation != selectedGeneration(v.srcGen) { |
| 1047 | return nil |
| 1048 | } |
| 1049 | m := r.m |
| 1050 | body, err := json.Marshal(genselect.Plan{Generation: plan.Generation, Files: plan.Files, Root: plan.Root, Pin: plan.Pin, MaxTXID: plan.MaxTXID, MaxLevel0TXID: plan.MaxLevel0TXID}) |
| 1051 | if err != nil { |
| 1052 | return err |
| 1053 | } |
| 1054 | m.mu.Lock() |
| 1055 | defer m.mu.Unlock() |
| 1056 | err = moves.UpdateJournal(m.DataDir, host, v.journalOp(), func(j *moves.Journal) error { |
| 1057 | if _, ok := j.Steps[rcStepSourceBaseline]; ok { |
| 1058 | return errJournalUnchanged |
| 1059 | } |
| 1060 | j.Mark(rcStepSourceBaseline, string(body)) |
| 1061 | return nil |
| 1062 | }) |
| 1063 | if errors.Is(err, errJournalUnchanged) { |
| 1064 | return nil |
| 1065 | } |
| 1066 | return err |
| 1067 | } |
| 1068 | |
| 1069 | // selectedGeneration is generation g as a selection names it: the claim |
| 1070 | // writes the origin as "" (a legacy source's src_gen) and genselect as |
| 1071 | // hostclaim.RootGeneration. |
| 1072 | func selectedGeneration(g string) string { |
| 1073 | if g == "" { |
| 1074 | return hostclaim.RootGeneration |
| 1075 | } |
| 1076 | return g |
| 1077 | } |
| 1078 | |
amadan