amadan
moves: the recovery wake owners, dispatched by kind (Plan 4 Task 9, Q7)
movesProbeOwners dispatches the observed op's kind: restoring and reverting go to the new movesRecoveryOwners, the landing stays, moving is refused. Every recovery admission is cold and keeps no stamp: a dirty return's first is into the return generation (sealed, the return seal's maximum, SelectPrepared over the journalled install manifest), a clean return's into a fresh generation from src_gen (clean-return), a declared or fenced return's and a revert's uncovered, a restart's a fresh generation from the certified remote (the return generation's pin first, H14). The inputs are exactly the recorded recovery inputs. AdmissionEvidence re-derives Select and refuses, typed, while a loss or fence is pending (Task 4 binds LossSources). Materialize renders and materializes the recorded inputs under their held works, publishes the return generation's verified marker, then runs C8's Establish (the new Files source; a fake in the tests until Task 6) and the start check. Publish, PublishFenced and PublicationEvidence work as the landing's; the landing's fenced publication and publication evidence are now shared helpers (publishFresh, publicationEvidence), unchanged in behaviour.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
3 files changed,
+1340
−27
internal/host/movesprobeowners.go+85 −27internal/host/movesrecoveryowners.go+604 −0internal/host/movesrecoveryowners_test.go+651 −0
diff --git a/internal/host/movesprobeowners.go b/internal/host/movesprobeowners.go| index ae575bcfb..d3f257f4b 100644 |
| --- a/internal/host/movesprobeowners.go |
| +++ b/internal/host/movesprobeowners.go |
| @@ -34,12 +34,16 @@ import ( |
| // The moves owners for a landing probe (moves Phase 1 Plan 3 Task 5c; claim |
| // Plan 9 Group D, D5; rulings H2, H4, H13, H14, H16, H18). The claim's |
| -// dispatcher (ClaimOwnerDispatch) hands this owner set only the move kinds; |
| -// of those it serves exactly one shape, a landing probe: the claim at |
| -// landing/probing with this box executing as the move's target, the policy |
| -// a probe, the registry row held by the move. Everything else is refused |
| -// with hostbox.ErrUnsupported or hostbox.ErrChanged, before any lease, stop |
| -// or launch: the committed wake is Task 7's, the recoveries Plan 4's. |
| +// dispatcher (ClaimOwnerDispatch) hands this owner set only the move kinds, |
| +// and it dispatches them again by the observed op's kind (ruling Q7): the |
| +// restoring and reverting kinds go to the recovery owners |
| +// (movesRecoveryOwners, movesrecoveryowners.go; Plan 4 Task 9), the moving |
| +// kind is refused, and the landing is served here. Of the landing it serves |
| +// one shape, a landing probe: the claim at landing/probing with this box |
| +// executing as the move's target, the policy a probe, the registry row held |
| +// by the move (and the committed wake, Task 7's, below). Everything else is |
| +// refused with hostbox.ErrUnsupported or hostbox.ErrChanged, before any |
| +// lease, stop or launch. |
| // |
| // The claim owns the admission, the restore, the launch, readiness, |
| // replication and the publication sequence (claimwakeassembly.go). This |
| @@ -133,6 +137,12 @@ type movesProbeOwners struct { |
| // (movespaging.go, §13): PageOperator once claim Plan 10 registers this |
| // owner set; nil, nothing pages. |
| Page func(ctx context.Context, level, subject, body string) error |
| + // Files is the moves C8 files manifest provider (claim §6.2a P6-D3; |
| + // Task 6 builds it). The recovery owners' Materialize establishes the |
| + // admitted generation's files manifest through it before every start; |
| + // without it every recovery kind is refused, typed, before anything is |
| + // read (movesrecoveryowners.go). |
| + Files ClaimFilesManifest |
| // pageWarned is the pages warned unsent for want of a pager. |
| pageWarned sync.Map |
| @@ -508,6 +518,9 @@ func probeEnv(env []string) []string { |
| // Select chooses the probe's launch inputs and admission. It is read-only. |
| func (m *movesProbeOwners) Select(ctx context.Context, rt registry.Route, o hostclaim.Observed, p activator.StartPolicy) (ClaimLaunchInputs, error) { |
| + if recoveryKind(o) { |
| + return m.recovery().Select(ctx, rt, o, p) |
| + } |
| v, err := m.landing(ctx, o) |
| if err != nil { |
| return ClaimLaunchInputs{}, err |
| @@ -589,6 +602,9 @@ func (m *movesProbeOwners) Select(ctx context.Context, rt registry.Route, o host |
| // AdmissionEvidence re-derives Select's admission against the observation |
| // and a baseline selected again, immediately before the admission's CAS. |
| func (m *movesProbeOwners) AdmissionEvidence(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, baseline *genselect.Plan) error { |
| + if recoveryKind(o) { |
| + return m.recovery().AdmissionEvidence(ctx, o, in, baseline) |
| + } |
| v, err := m.landing(ctx, o) |
| if err != nil { |
| return err |
| @@ -691,6 +707,9 @@ func (w heldIdentityWork) With(ctx context.Context, _ hostwrite.Identity, fn fun |
| // verified prepared marker under the admitted writer's fence, then the |
| // start check. The spawn follows only if all of it passed. |
| func (m *movesProbeOwners) Materialize(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { |
| + if recoveryKind(o) { |
| + return m.recovery().Materialize(ctx, o, in, work) |
| + } |
| v, err := m.landing(ctx, o) |
| if err != nil { |
| return err |
| @@ -909,8 +928,12 @@ func (m *movesProbeOwners) readMarker(ctx context.Context, h *hostclaim.Holder) |
| return &marker, nil |
| } |
| -// EmptyAuthority: a move never admits the empty bootstrap. |
| -func (m *movesProbeOwners) EmptyAuthority(context.Context, hostclaim.Observed, ClaimLaunchInputs) error { |
| +// EmptyAuthority: a move never admits the empty bootstrap, a recovery |
| +// included. |
| +func (m *movesProbeOwners) EmptyAuthority(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { |
| + if recoveryKind(o) { |
| + return m.recovery().EmptyAuthority(ctx, o, in) |
| + } |
| return hostbox.ErrUnsupported |
| } |
| @@ -919,6 +942,9 @@ func (m *movesProbeOwners) EmptyAuthority(context.Context, hostclaim.Observed, C |
| // finds it and records the receipt. It writes only under the fence |
| // Materialize recorded for the admitted session, and refuses without one. |
| func (m *movesProbeOwners) Publish(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { |
| + if recoveryKind(o) { |
| + return m.recovery().Publish(ctx, o, in) |
| + } |
| v, err := m.landing(ctx, o) |
| if err != nil { |
| return err |
| @@ -930,9 +956,7 @@ func (m *movesProbeOwners) Publish(ctx context.Context, o hostclaim.Observed, in |
| if !in.Prepared || in.Admission.NewGeneration != v.op { |
| return hostbox.ErrChanged |
| } |
| - m.mu.Lock() |
| - work := m.fences[claimPublishedKey(o.Host, h.Session)] |
| - m.mu.Unlock() |
| + work := m.fence(o.Host, h.Session) |
| if work == nil { |
| return fmt.Errorf("%w: no admitted writer's fence for session %d", hostbox.ErrChanged, h.Session) |
| } |
| @@ -951,6 +975,9 @@ func (m *movesProbeOwners) Publish(ctx context.Context, o hostclaim.Observed, in |
| // the session's own first root, under the publisher's fence, as the claim's |
| // serving publisher does (claimServingOwners.PublishFenced). |
| func (m *movesProbeOwners) PublishFenced(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, pub ClaimPublication) (ClaimPublicationReceipt, error) { |
| + if recoveryKind(o) { |
| + return m.recovery().PublishFenced(ctx, o, in, pub) |
| + } |
| v, err := m.landing(ctx, o) |
| if err != nil { |
| return ClaimPublicationReceipt{}, err |
| @@ -967,9 +994,7 @@ func (m *movesProbeOwners) PublishFenced(ctx context.Context, o hostclaim.Observ |
| } |
| // The admitted writer's spawn selection, recorded by Materialize: the |
| // fence the local files marker is refreshed under (fix round 1). |
| - m.mu.Lock() |
| - work := m.fences[claimPublishedKey(o.Host, h.Session)] |
| - m.mu.Unlock() |
| + work := m.fence(o.Host, h.Session) |
| if work == nil { |
| return ClaimPublicationReceipt{}, fmt.Errorf("%w: no admitted writer's fence for session %d", hostbox.ErrChanged, h.Session) |
| } |
| @@ -990,12 +1015,24 @@ func (m *movesProbeOwners) PublishFenced(ctx context.Context, o hostclaim.Observ |
| if err := m.recordFilesDigest(o.Host, v.op, h.Generation, digest); err != nil { |
| return ClaimPublicationReceipt{}, err |
| } |
| + return m.publishFresh(ctx, o.Host, h, pub, func() error { return m.refreshFilesGen(ctx, o.Host, v, work, in.Identity, digest) }) |
| +} |
| + |
| +// publishFresh is the fenced publication of the holder's fresh generation, |
| +// shared by the landing's restarted attempt and the recovery owners: the |
| +// session's replicator synced, the generation's own first root selected, |
| +// then generation.json naming the generation superseding the current marker |
| +// by its exact ETag (created when there is none), pub.Check immediately |
| +// before the write and before every attempt. A replay that finds this |
| +// generation's marker from this box records it. refresh runs after the |
| +// marker, before the receipt is recorded (the local files marker). |
| +func (m *movesProbeOwners) publishFresh(ctx context.Context, host string, h *hostclaim.Holder, pub ClaimPublication, refresh func() error) (ClaimPublicationReceipt, error) { |
| if err := pub.Sync(ctx); err != nil { |
| return ClaimPublicationReceipt{}, err |
| } |
| bounded, cancel := context.WithTimeout(ctx, hostbox.RequestLimit) |
| defer cancel() |
| - plan, err := genselect.Select(bounded, m.Data, m.request(o.Host, h)) |
| + plan, err := genselect.Select(bounded, m.Data, m.request(host, h)) |
| if err != nil { |
| return ClaimPublicationReceipt{}, err |
| } |
| @@ -1014,13 +1051,13 @@ func (m *movesProbeOwners) PublishFenced(ctx context.Context, o hostclaim.Observ |
| return ClaimPublicationReceipt{}, hostbox.ErrChanged |
| } |
| receipt := ClaimPublicationReceipt{MarkerVersion: current.ID, Root: root} |
| - if err := m.refreshFilesGen(ctx, o.Host, v, work, in.Identity, digest); err != nil { |
| + if err := refresh(); err != nil { |
| return ClaimPublicationReceipt{}, err |
| } |
| - m.record(o.Host, h.Session, receipt) |
| + m.record(host, h.Session, receipt) |
| return receipt, nil |
| } |
| - body, err := json.Marshal(genselect.Marker{Op: h.Generation, Incarnation: h.Incarnation, Host: o.Host, Box: m.Box, InstalledAt: m.Now().UTC()}) |
| + body, err := json.Marshal(genselect.Marker{Op: h.Generation, Incarnation: h.Incarnation, Host: host, Box: m.Box, InstalledAt: m.Now().UTC()}) |
| if err != nil { |
| return ClaimPublicationReceipt{}, err |
| } |
| @@ -1039,10 +1076,10 @@ func (m *movesProbeOwners) PublishFenced(ctx context.Context, o hostclaim.Observ |
| return ClaimPublicationReceipt{}, hostwrite.ErrUnavailable |
| } |
| receipt := ClaimPublicationReceipt{MarkerVersion: version, Root: root} |
| - if err := m.refreshFilesGen(ctx, o.Host, v, work, in.Identity, digest); err != nil { |
| + if err := refresh(); err != nil { |
| return ClaimPublicationReceipt{}, err |
| } |
| - m.record(o.Host, h.Session, receipt) |
| + m.record(host, h.Session, receipt) |
| return receipt, nil |
| } |
| @@ -1058,6 +1095,17 @@ func (m *movesProbeOwners) recordFence(host string, session uint64, work hostwri |
| m.fences[claimPublishedKey(host, session)] = work |
| } |
| +// fence is the admitted spawn selection Materialize recorded for the |
| +// session, or nil. |
| +func (m *movesProbeOwners) fence(host string, session uint64) hostwrite.Work { |
| + m.mu.Lock() |
| + defer m.mu.Unlock() |
| + return m.fences[claimPublishedKey(host, session)] |
| +} |
| + |
| +// recovery is the recovery owners over this owner set's sources. |
| +func (m *movesProbeOwners) recovery() movesRecoveryOwners { return movesRecoveryOwners{m: m} } |
| + |
| // verifyFiles requires the files directory and <host>.env on disk to be the |
| // ones the landing placed: the install receipt's identities |
| // (landStepFilesPlaced). Anything else fails the publication. |
| @@ -1207,34 +1255,44 @@ func (m *movesProbeOwners) record(host string, session uint64, receipt ClaimPubl |
| // holder's generation, incarnation, host and this box (and, for the move |
| // id, the build's pin), and the root it named still exists, current. |
| func (m *movesProbeOwners) PublicationEvidence(ctx context.Context, o hostclaim.Observed) error { |
| + if recoveryKind(o) { |
| + return m.recovery().PublicationEvidence(ctx, o) |
| + } |
| v, err := m.landing(ctx, o) |
| if err != nil { |
| return err |
| } |
| - h := v.holder |
| + return m.publicationEvidence(ctx, o.Host, v.holder, v.op, v.manifest) |
| +} |
| + |
| +// publicationEvidence is PublicationEvidence's check of the admitted |
| +// session's receipt, shared with the recovery owners. prepared names the |
| +// prepared generation whose marker also carries manifest's pin and digest |
| +// ("" for none). |
| +func (m *movesProbeOwners) publicationEvidence(ctx context.Context, host string, h *hostclaim.Holder, prepared string, manifest moves.InstallManifest) error { |
| if h.WriterBox != m.Box { |
| return hostbox.ErrChanged |
| } |
| m.mu.Lock() |
| - receipt, ok := m.published[claimPublishedKey(o.Host, h.Session)] |
| + receipt, ok := m.published[claimPublishedKey(host, h.Session)] |
| m.mu.Unlock() |
| prefix, err := hostbox.GenerationPrefix(*h) |
| if !ok || err != nil || receipt.MarkerVersion == "" || receipt.Root.VersionID == "" || !strings.HasPrefix(receipt.Root.Key, prefix) { |
| return hostbox.ErrChanged |
| } |
| - plan, err := genselect.Select(ctx, m.Data, m.request(o.Host, h)) |
| + plan, err := genselect.Select(ctx, m.Data, m.request(host, h)) |
| if err != nil { |
| return err |
| } |
| - if plan.Generation != h.Generation || plan.Marker == nil || plan.MarkerVersion != receipt.MarkerVersion || plan.Marker.Op != h.Generation || plan.Marker.Box != m.Box || plan.Marker.Incarnation != h.Incarnation || plan.Marker.Host != o.Host { |
| + if plan.Generation != h.Generation || plan.Marker == nil || plan.MarkerVersion != receipt.MarkerVersion || plan.Marker.Op != h.Generation || plan.Marker.Box != m.Box || plan.Marker.Incarnation != h.Incarnation || plan.Marker.Host != host { |
| return hostbox.ErrChanged |
| } |
| - if h.Generation == v.op { |
| - digest, err := manifestDigest(v.manifest) |
| + if prepared != "" && h.Generation == prepared { |
| + digest, err := manifestDigest(manifest) |
| if err != nil { |
| return err |
| } |
| - if plan.Marker.Pin != manifestPin(v.manifest) || plan.Marker.ManifestSHA256 != digest { |
| + if plan.Marker.Pin != manifestPin(manifest) || plan.Marker.ManifestSHA256 != digest { |
| return hostbox.ErrChanged |
| } |
| } |
diff --git a/internal/host/movesrecoveryowners.go b/internal/host/movesrecoveryowners.go| new file mode 100644 |
| index 000000000..c883d75cf |
| --- /dev/null |
| +++ b/internal/host/movesrecoveryowners.go |
| @@ -0,0 +1,604 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "fmt" |
| + "path/filepath" |
| + "reflect" |
| + "strconv" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/genselect" |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| + "github.com/carlosframework/platform/internal/hostwrite" |
| + "github.com/carlosframework/platform/internal/moves" |
| + "github.com/carlosframework/platform/internal/moves/archive" |
| + "github.com/carlosframework/platform/internal/park" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| +) |
| + |
| +// The moves owners for a recovery probe (moves Phase 1 Plan 4 Task 9; |
| +// rulings Q7 and Q6 of the moves-after-Plan-12 preflight, J2, J14, H14). The |
| +// moves owner set (movesProbeOwners) dispatches by the observed op's kind: |
| +// a landing to the landing probe's owners, restoring and reverting here. |
| +// |
| +// One shape is served: the source's recovery probe, the claim restoring (a |
| +// rollback's return) or reverting (rev_phase probing) at op.phase probing, |
| +// this box executing as the move's source in the move's source region, not |
| +// the target region and not lost; the record this move and this holder |
| +// incarnation, rolling_back for the claim's rollback at source phase probing |
| +// or reverting and not reopened; the row held by the rollback id (restoring) |
| +// or the move id (reverting). Everything else is refused before any lease, |
| +// stop or launch. |
| +// |
| +// Every recovery admission is cold and keeps no stamp (J14: S is retired by |
| +// the first recovery admission). The first one is told by the holder still |
| +// at the source's stamp S (op.journal.source_stamp, which ReturnToSource |
| +// and Revert leave in holder.stamp); any later one is a restart. |
| +// |
| +// - Select (read-only). A dirty return's first admission is into the |
| +// return generation (the rollback id) the source's install built, after |
| +// PreparedCheck, sealed when the return seal was built from the stopped |
| +// database, its baseline the return generation's pin at the return |
| +// seal's maximum (genselect.SelectPrepared). A clean return's first is |
| +// into a fresh generation restored from src_gen (the holder's generation |
| +// ReturnToSource named), coverage clean-return; a declared or fenced |
| +// return asks no coverage (Q6). A revert's first is into a fresh |
| +// generation restored from the holder's own generation, which a revert |
| +// never changed, with no coverage. A restart is a fresh generation with |
| +// no coverage, restored from the certified remote (genselect.Select), or, |
| +// for a dirty return while the location's marker is not yet the return's, |
| +// the return generation's pin (H14). The launch inputs are exactly the |
| +// recorded ones (Rollback.RecoveryInputs or Move.RecoveryInputs; the |
| +// start policy carries them), the live release and the launch |
| +// environment's report are theirs, and CARLOS_MOVE_PROBE=1 is in the |
| +// child's environment, as for the landing's probe. |
| +// - AdmissionEvidence: the same decisions re-derived and compared against |
| +// a baseline selected again. While a declared loss or a target fence is |
| +// pending, the writer replaces RemoteMaxTXID with the frozen |
| +// predecessor's maximum from its LossSources call (writer_loss.go), which |
| +// this owner cannot vouch for until that contract is bound (Task 4): it |
| +// refuses, typed. |
| +// - Materialize, before every start: the held render and the held release |
| +// of the recorded inputs, each under its own claim selection at the |
| +// admitted capture (Works: held-config, held-release; MovesClaimPolicy's |
| +// recovery rows bind them to the recorded inputs), the launch |
| +// environment's report again, then, for a dirty return, the return |
| +// generation's verified marker under the admitted writer's fence, then |
| +// C8's Establish under the start's work, then the §5.1 start check. |
| +// - Publish, PublishFenced and PublicationEvidence work as the landing's: |
| +// the dirty return's prepared marker before replication, a fresh |
| +// generation's marker after its own first root, and this process's |
| +// receipt read back. |
| +// |
| +// Production: nothing registers this owner set yet (Plan 5 Task 9b). C8 is |
| +// Task 6's provider; without one every recovery kind is refused, typed. |
| +type movesRecoveryOwners struct{ m *movesProbeOwners } |
| + |
| +// recoveryKind reports whether o is a recovery the recovery owners serve. |
| +func recoveryKind(o hostclaim.Observed) bool { |
| + return !o.Absent && o.Claim != nil && claimRecoveryKind(o.Claim.Op.Kind) |
| +} |
| + |
| +// recoveryProbe is what one recovery probe's checks read. |
| +type recoveryProbe struct { |
| + homeRegion string |
| + kind string |
| + // op is the move id; holdOp the op holding the row: the rollback id for |
| + // a return, the move id for a revert. |
| + op, holdOp string |
| + holder *hostclaim.Holder |
| + rec *provision.Record |
| + // inputs are the recorded recovery inputs. |
| + inputs *provision.MoveInputs |
| + // first: the holder is still at the source's stamp S, so no recovery |
| + // admission has happened yet. |
| + first bool |
| + // dirty: a dirty return. uncovered: a declared or fenced return (Q6). |
| + dirty, uncovered bool |
| + srcGen string |
| + predecessor string |
| + // A dirty return's: the rollback id (its generation), the return seal |
| + // and the install manifest its install journalled. |
| + rb string |
| + seal moves.Seal |
| + manifest moves.InstallManifest |
| +} |
| + |
| +// prepared is the dirty return's build as the landing's helpers read one: |
| +// the return generation is the "op" whose verified marker is published. |
| +func (v recoveryProbe) prepared() probeLanding { |
| + return probeLanding{homeRegion: v.homeRegion, op: v.rb, holder: v.holder, rec: v.rec, seal: v.seal, manifest: v.manifest} |
| +} |
| + |
| +// recoveryLossContract is the typed refusal while the recovery's loss and |
| +// fence admission evidence is unbound (Task 4 binds LossSources). |
| +func recoveryLossContract() error { |
| + return &ClaimDependencyError{Contracts: []string{"moves recovery loss and fence admission evidence (LossSources)"}} |
| +} |
| + |
| +// recoveryInputsChanged is a launch input that is not the recorded one: the |
| +// next pass records the inputs again and probes again. |
| +func recoveryInputsChanged(what string) error { |
| + return fmt.Errorf("%w: the recovery's %s is not the recorded one", hostbox.ErrChanged, what) |
| +} |
| + |
| +// recovery checks that o is a recovery probe this box may run and reads the |
| +// record, the recorded inputs and, for a dirty return, the return seal and |
| +// the journalled install manifest. |
| +func (r movesRecoveryOwners) recovery(ctx context.Context, o hostclaim.Observed) (recoveryProbe, error) { |
| + m := r.m |
| + var v recoveryProbe |
| + if !m.valid() { |
| + return v, hostwrite.ErrUnavailable |
| + } |
| + if m.Files == nil { |
| + return v, claimFilesRefusal() |
| + } |
| + if !recoveryKind(o) || o.Claim.Holder == nil { |
| + return v, hostbox.ErrUnsupported |
| + } |
| + c := o.Claim |
| + home, err := park.VerifiedHome(m.FleetHomeRegion, m.HomeRegion, m.Region, c.Op.Intent["home_region"]) |
| + if err != nil { |
| + return v, err |
| + } |
| + v.homeRegion, v.kind = home, c.Op.Kind |
| + if c.Op.Phase != hostclaim.PhaseProbing || (v.kind == hostclaim.KindReverting && c.Op.Journal[hostclaim.JournalRevPhase] != hostclaim.PhaseProbing) { |
| + return v, hostbox.ErrChanged |
| + } |
| + if c.Host != o.Host || c.Op.ID == "" || c.Op.Exec != m.Box || len(c.Op.Participants) == 0 || c.Op.Participants[0] != m.Box || |
| + c.Op.Intent["source"] != m.Box || c.Op.Intent["target"] == m.Box || park.SameRegion(m.HomeRegion, c.Op.Intent["to"], m.Region) { |
| + return v, hostbox.ErrChanged |
| + } |
| + for _, lost := range c.Lost { |
| + if lost == m.Box { |
| + return v, hostbox.ErrChanged |
| + } |
| + } |
| + v.op, v.holder = c.Op.ID, c.Holder |
| + stamp, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64) |
| + srcGen, journalled := c.Op.Journal[hostclaim.JournalSrcGen] |
| + if err != nil || stamp == 0 || !journalled { |
| + return v, fmt.Errorf("%w: the source's stamp and generation are not journalled", hostbox.ErrChanged) |
| + } |
| + v.first, v.srcGen = v.holder.Stamp == stamp, srcGen |
| + rec, err := m.Records(ctx, v.holder.Account, v.holder.App, o.Host) |
| + if err != nil { |
| + return v, err |
| + } |
| + if rec == nil || rec.Host != o.Host || rec.Move == nil { |
| + return v, hostbox.ErrChanged |
| + } |
| + mv := rec.Move |
| + if mv.ID != v.op || mv.Incarnation != v.holder.Incarnation || !park.PlacementMatches(m.HomeRegion, mv.From, m.Region) { |
| + return v, hostbox.ErrChanged |
| + } |
| + switch v.kind { |
| + case hostclaim.KindRestoring: |
| + rb := mv.Rollback |
| + if mv.State != provision.MoveRollingBack || rb == nil || rb.ID == "" || rb.ID != c.Op.Intent["rollback"] || rb.SourcePhase != provision.RbSourceProbing { |
| + return v, hostbox.ErrChanged |
| + } |
| + v.rb, v.holdOp, v.dirty, v.inputs = rb.ID, rb.ID, rb.Dirty, rb.RecoveryInputs |
| + lossID, err := hostclaim.OperationLossID(c) |
| + if err != nil { |
| + return v, err |
| + } |
| + claimed := lossID != "" || c.Op.Journal["fence."+c.Op.Intent["target"]] != "" |
| + if claimed != (rb.LossRef != "") { |
| + // The record and the claim disagree on the target's loss: wait. |
| + return v, fmt.Errorf("%w: the record and the claim disagree on the target's loss", hostbox.ErrChanged) |
| + } |
| + v.uncovered = claimed |
| + if v.dirty && v.uncovered { |
| + return v, fmt.Errorf("%w: a declared or fenced return is never dirty", hostbox.ErrChanged) |
| + } |
| + default: // hostclaim.KindReverting |
| + if mv.State != provision.MoveReverting || mv.ReopenedAt != nil { |
| + return v, hostbox.ErrChanged |
| + } |
| + v.holdOp, v.inputs = v.op, mv.RecoveryInputs |
| + } |
| + if v.first && !v.dirty && v.holder.Generation != v.srcGen { |
| + // A clean return names src_gen again (ReturnToSource); a revert |
| + // never left it. |
| + return v, fmt.Errorf("%w: the holder is not at the source's generation", hostbox.ErrChanged) |
| + } |
| + if v.inputs == nil || v.inputs.Release == "" { |
| + return v, fmt.Errorf("%w: no recorded recovery inputs", hostbox.ErrChanged) |
| + } |
| + if v.predecessor, err = predecessorRegion(v.holder, mv.From); err != nil { |
| + return v, err |
| + } |
| + v.rec = rec |
| + if !v.dirty { |
| + return v, nil |
| + } |
| + if v.seal, err = returnSeal(c); err != nil { |
| + return v, err |
| + } |
| + j, err := moves.ReadJournal(m.DataDir, o.Host, v.rb) |
| + if err != nil { |
| + return v, err |
| + } |
| + raw, ok := j.Steps[landStepInstall] |
| + if !ok { |
| + return v, fmt.Errorf("%w: the return journalled no install manifest", hostbox.ErrChanged) |
| + } |
| + if err := json.Unmarshal([]byte(raw), &v.manifest); err != nil { |
| + return v, fmt.Errorf("the journalled install manifest: %w", err) |
| + } |
| + if v.manifest.Op != v.rb || v.manifest.Host != o.Host || v.manifest.N != v.seal.N || v.manifest.N == 0 || v.manifest.SnapshotKey == "" || v.manifest.SnapshotVersion == "" { |
| + return v, fmt.Errorf("%w: the install manifest is not this return's build at the seal", hostbox.ErrChanged) |
| + } |
| + return v, nil |
| +} |
| + |
| +// recoveryLossPending reports a declared loss or a target fence the writer |
| +// is about to admit (writer.go's routing): an unconsumed operation loss, or |
| +// the consumed fence's frozen target writer before its first admission. |
| +func recoveryLossPending(c *hostclaim.Claim) (bool, error) { |
| + if _, pending := hostclaim.PendingMoveFenceLoss(c); pending { |
| + return true, nil |
| + } |
| + id, err := hostclaim.OperationLossID(c) |
| + if err != nil { |
| + return false, err |
| + } |
| + return id != "" && !hostclaim.OperationLossConsumed(c, id), nil |
| +} |
| + |
| +// preparedAdmission: a dirty return's first admission, into the return |
| +// generation. |
| +func (v recoveryProbe) preparedAdmission() bool { return v.dirty && v.first } |
| + |
| +// freshGeneration: g may name a recovery's fresh generation. |
| +func (v recoveryProbe) freshGeneration(g string) bool { |
| + return g != v.op && (v.rb == "" || g != v.rb) && claimFreshGeneration(v.holder, g) |
| +} |
| + |
| +// admission is the recovery's admission and baseline for v, the new |
| +// generation fresh unless it is the dirty return's first (see the file |
| +// comment). dbPath is the held row's database, for PreparedCheck. |
| +func (r movesRecoveryOwners) admission(ctx context.Context, o hostclaim.Observed, v recoveryProbe, dbPath, fresh string) (hostclaim.Admission, *genselect.Plan, error) { |
| + m := r.m |
| + h := v.holder |
| + a := hostclaim.Admission{ObservedOpID: v.op, ObservedPhase: o.Claim.Op.Phase, ObservedSession: h.Session, ObservedGeneration: h.Generation, Box: m.Box, Mode: "cold", |
| + Region: m.Region, PredecessorRegion: v.predecessor} |
| + var ( |
| + plan *genselect.Plan |
| + err error |
| + ) |
| + if v.preparedAdmission() { |
| + files, db, err := m.localMarkers(o.Host, dbPath) |
| + if err != nil { |
| + return a, nil, err |
| + } |
| + if err := moves.PreparedCheck(v.rb, *h, files, db); err != nil { |
| + return a, nil, err |
| + } |
| + if !claimFreshGeneration(h, v.rb) { |
| + return a, nil, hostbox.ErrChanged |
| + } |
| + a.NewGeneration = v.rb |
| + if v.seal.FromDB { |
| + a.Coverage = recoveryCoverageSealed |
| + } |
| + if plan, err = genselect.SelectPrepared(ctx, m.Data, m.request(o.Host, h), preparedPin(v.prepared())); err != nil { |
| + return a, nil, err |
| + } |
| + // The maximum is the return seal's. |
| + if plan.MaxTXID != v.seal.N || plan.Pin == nil || *plan.Pin != manifestPin(v.manifest) { |
| + return a, nil, fmt.Errorf("%w: the return generation is not at the return seal", hostbox.ErrChanged) |
| + } |
| + } else { |
| + a.NewGeneration = fresh |
| + if v.first && v.kind == hostclaim.KindRestoring && !v.uncovered { |
| + a.Coverage = recoveryCoverageClean |
| + } |
| + if plan, err = r.baseline(ctx, o.Host, v); err != nil { |
| + return a, nil, err |
| + } |
| + } |
| + a.RemoteMaxTXID = plan.MaxTXID |
| + return a, plan, nil |
| +} |
| + |
| +// baseline is a fresh generation's baseline: the certified remote; for a |
| +// dirty return's restart while the location's marker is not yet the return |
| +// generation's (nor a generation descending from it on this box), the |
| +// return generation's pin (H14). |
| +func (r movesRecoveryOwners) baseline(ctx context.Context, host string, v recoveryProbe) (*genselect.Plan, error) { |
| + m := r.m |
| + if v.dirty { |
| + digest, err := manifestDigest(v.manifest) |
| + if err != nil { |
| + return nil, err |
| + } |
| + marker, err := m.readMarker(ctx, v.holder) |
| + if err != nil { |
| + return nil, err |
| + } |
| + if !m.landingMarker(marker, host, v.prepared(), digest) { |
| + return genselect.SelectPrepared(ctx, m.Data, m.request(host, v.holder), preparedPin(v.prepared())) |
| + } |
| + } |
| + return genselect.Select(ctx, m.Data, m.request(host, v.holder)) |
| +} |
| + |
| +// Select chooses the recovery probe's launch inputs and admission. It is |
| +// read-only. |
| +func (r movesRecoveryOwners) Select(ctx context.Context, rt registry.Route, o hostclaim.Observed, p activator.StartPolicy) (ClaimLaunchInputs, error) { |
| + m := r.m |
| + v, err := r.recovery(ctx, o) |
| + if err != nil { |
| + return ClaimLaunchInputs{}, err |
| + } |
| + if !p.Probe { |
| + return ClaimLaunchInputs{}, hostbox.ErrUnsupported |
| + } |
| + if p.Inputs == nil || !sameInputs(v.inputs, *p.Inputs) || (p.Prepared != nil && (!v.dirty || *p.Prepared != v.manifest)) { |
| + return ClaimLaunchInputs{}, hostbox.ErrChanged |
| + } |
| + if err := heldRoute(rt, o.Host, v.holdOp); err != nil { |
| + return ClaimLaunchInputs{}, err |
| + } |
| + fresh := "" |
| + if !v.preparedAdmission() { |
| + if fresh = m.NewID(); !v.freshGeneration(fresh) { |
| + return ClaimLaunchInputs{}, hostwrite.ErrUnavailable |
| + } |
| + } |
| + admission, plan, err := r.admission(ctx, o, v, rt.DBPath, fresh) |
| + if err != nil { |
| + return ClaimLaunchInputs{}, err |
| + } |
| + h := v.holder |
| + ready, err := m.ReadyPath(ctx, h.Account, h.App) |
| + if err != nil { |
| + return ClaimLaunchInputs{}, err |
| + } |
| + l, err := m.Launch(ctx, rt) |
| + if err != nil { |
| + return ClaimLaunchInputs{}, err |
| + } |
| + if !filepath.IsAbs(l.Executable) || !claimHexDigest(l.ExecutableSHA256) || l.Version == "" { |
| + return ClaimLaunchInputs{}, hostwrite.ErrUnavailable |
| + } |
| + if l.Version != v.inputs.Release { |
| + return ClaimLaunchInputs{}, recoveryInputsChanged("live release") |
| + } |
| + if v.inputs.EnvReport == "" || envDigest(m.EnvKey, l.Env) != v.inputs.EnvReport { |
| + return ClaimLaunchInputs{}, recoveryInputsChanged("launch environment") |
| + } |
| + return ClaimLaunchInputs{Roles: []string{"main"}, Identity: launchIdentity(v.inputs), Baseline: plan, Admission: admission, |
| + Executable: l.Executable, ExecutableSHA256: l.ExecutableSHA256, Args: append([]string(nil), l.Args...), Env: probeEnv(l.Env), |
| + DBPath: rt.DBPath, Version: l.Version, Limit: l.Limit, Prepared: v.preparedAdmission(), ReadyPath: ready}, nil |
| +} |
| + |
| +// AdmissionEvidence re-derives Select's admission against the observation |
| +// and a baseline selected again, immediately before the admission's CAS. |
| +func (r movesRecoveryOwners) AdmissionEvidence(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, baseline *genselect.Plan) error { |
| + m := r.m |
| + v, err := r.recovery(ctx, o) |
| + if err != nil { |
| + return err |
| + } |
| + // The writer substitutes the frozen maximum under a pending loss or |
| + // fence; only the loss sources vouch for it (Task 4). |
| + pending, err := recoveryLossPending(o.Claim) |
| + if err != nil { |
| + return err |
| + } |
| + if pending { |
| + return recoveryLossContract() |
| + } |
| + if baseline == nil || in.Mode != "cold" || in.PredecessorCertified || in.LossID != "" || len(in.AdditionalWritesLost) != 0 || in.KeepStamp || in.RemoteMaxTXID != baseline.MaxTXID { |
| + return hostbox.ErrChanged |
| + } |
| + if !v.preparedAdmission() && !v.freshGeneration(in.NewGeneration) { |
| + return hostbox.ErrChanged |
| + } |
| + rt, err := m.route(o.Host, v.holdOp) |
| + if err != nil { |
| + return err |
| + } |
| + want, fresh, err := r.admission(ctx, o, v, rt.DBPath, in.NewGeneration) |
| + if err != nil { |
| + return err |
| + } |
| + if in.ObservedOpID != want.ObservedOpID || in.ObservedPhase != want.ObservedPhase || in.ObservedSession != want.ObservedSession || in.ObservedGeneration != want.ObservedGeneration || |
| + in.Box != want.Box || !park.SameRegion(m.HomeRegion, in.Region, m.Region) || !park.SameRegion(v.homeRegion, in.PredecessorRegion, want.PredecessorRegion) || |
| + in.NewGeneration != want.NewGeneration || in.Coverage != want.Coverage { |
| + return hostbox.ErrChanged |
| + } |
| + if fresh.Generation != baseline.Generation || fresh.MaxTXID != baseline.MaxTXID || !reflect.DeepEqual(fresh.Files, baseline.Files) || !reflect.DeepEqual(fresh.Pin, baseline.Pin) { |
| + return hostbox.ErrChanged |
| + } |
| + return nil |
| +} |
| + |
| +// Materialize runs before every start of the admitted recovery probe, in the |
| +// ruled order (see the file comment). The spawn follows only if all of it |
| +// passed. |
| +func (r movesRecoveryOwners) Materialize(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { |
| + m := r.m |
| + v, err := r.recovery(ctx, o) |
| + if err != nil { |
| + return err |
| + } |
| + if err := admittedProbe(o, in, m.Box); err != nil { |
| + return err |
| + } |
| + if work == nil { |
| + return hostwrite.ErrUnavailable |
| + } |
| + if in.Identity != launchIdentity(v.inputs) { |
| + return hostbox.ErrChanged |
| + } |
| + fence := writerFence(work, in.Identity) |
| + if err := fence(ctx); err != nil { |
| + return err |
| + } |
| + rt, err := m.route(o.Host, v.holdOp) |
| + if err != nil { |
| + return err |
| + } |
| + if rt.DBPath != in.DBPath { |
| + return hostbox.ErrChanged |
| + } |
| + h := v.holder |
| + captured := hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp} |
| + hr, err := m.Render(ctx, rt, v.holdOp, func(pre appconfig.HeldRender) (hostwrite.Work, error) { |
| + // The recorded inputs decide before any selection or write. |
| + if !sameBundle(pre.BundleETags, v.inputs.BundleETags) || pre.OverlayETag != v.inputs.OverlayETag { |
| + return nil, recoveryInputsChanged("configuration") |
| + } |
| + w, err := m.Works(ctx, o.Host, movesActionHeldConfig, captured) |
| + if err != nil { |
| + return nil, err |
| + } |
| + return heldIdentityWork{Work: w, ID: HeldConfigIdentity(pre.BundleETags, pre.OverlayETag)}, nil |
| + }) |
| + if err != nil { |
| + return err |
| + } |
| + if !sameBundle(hr.BundleETags, v.inputs.BundleETags) || hr.OverlayETag != v.inputs.OverlayETag { |
| + return recoveryInputsChanged("configuration") |
| + } |
| + released, err := m.Works(ctx, o.Host, movesActionHeldRelease, captured) |
| + if err != nil { |
| + return err |
| + } |
| + if err := m.Release(ctx, rt, v.inputs.Release, heldIdentityWork{Work: released, ID: hostwrite.Identity{Release: v.inputs.Release}}); err != nil { |
| + return err |
| + } |
| + if envDigest(m.EnvKey, in.Env) != v.inputs.EnvReport { |
| + return recoveryInputsChanged("launch environment") |
| + } |
| + m.recordFence(o.Host, h.Session, work) |
| + if v.dirty { |
| + receipt, err := m.preparedMarker(ctx, o.Host, v.prepared(), fence) |
| + if err != nil { |
| + return err |
| + } |
| + if in.Prepared { |
| + m.record(o.Host, h.Session, receipt) |
| + } |
| + } |
| + if err := m.Files.Establish(ctx, o, in, work); err != nil { |
| + return err |
| + } |
| + return m.startCheck(ctx, o, in) |
| +} |
| + |
| +// EmptyAuthority: a recovery never admits the empty bootstrap. |
| +func (r movesRecoveryOwners) EmptyAuthority(context.Context, hostclaim.Observed, ClaimLaunchInputs) error { |
| + return hostbox.ErrUnsupported |
| +} |
| + |
| +// Publish is Group D's call for the dirty return's prepared admission, |
| +// before replication: Materialize published the return generation's |
| +// verified marker; this finds it under the admitted writer's fence and |
| +// records the receipt. |
| +func (r movesRecoveryOwners) Publish(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs) error { |
| + m := r.m |
| + v, err := r.recovery(ctx, o) |
| + if err != nil { |
| + return err |
| + } |
| + if err := admittedProbe(o, in, m.Box); err != nil { |
| + return err |
| + } |
| + h := v.holder |
| + if !v.dirty || !in.Prepared || in.Admission.NewGeneration != v.rb || h.Generation != v.rb { |
| + return hostbox.ErrChanged |
| + } |
| + work := m.fence(o.Host, h.Session) |
| + if work == nil { |
| + return fmt.Errorf("%w: no admitted writer's fence for session %d", hostbox.ErrChanged, h.Session) |
| + } |
| + receipt, err := m.preparedMarker(ctx, o.Host, v.prepared(), writerFence(work, in.Identity)) |
| + if err != nil { |
| + return err |
| + } |
| + if receipt.MarkerVersion == "" { |
| + return hostbox.ErrChanged |
| + } |
| + m.record(o.Host, h.Session, receipt) |
| + return nil |
| +} |
| + |
| +// PublishFenced writes a fresh recovery generation's marker after the |
| +// session's own first root, under the publisher's fence, as the landing's |
| +// restarted attempt and serving do. |
| +func (r movesRecoveryOwners) PublishFenced(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, pub ClaimPublication) (ClaimPublicationReceipt, error) { |
| + m := r.m |
| + v, err := r.recovery(ctx, o) |
| + if err != nil { |
| + return ClaimPublicationReceipt{}, err |
| + } |
| + if err := admittedProbe(o, in, m.Box); err != nil { |
| + return ClaimPublicationReceipt{}, err |
| + } |
| + if pub.Sync == nil || pub.Check == nil { |
| + return ClaimPublicationReceipt{}, hostwrite.ErrUnavailable |
| + } |
| + h := v.holder |
| + if in.Prepared || h.Generation == v.op || (v.rb != "" && h.Generation == v.rb) { |
| + return ClaimPublicationReceipt{}, hostbox.ErrChanged |
| + } |
| + work := m.fence(o.Host, h.Session) |
| + if work == nil { |
| + return ClaimPublicationReceipt{}, fmt.Errorf("%w: no admitted writer's fence for session %d", hostbox.ErrChanged, h.Session) |
| + } |
| + return m.publishFresh(ctx, o.Host, h, pub, func() error { return r.refreshFilesGen(ctx, o.Host, h, work, in.Identity) }) |
| +} |
| + |
| +// refreshFilesGen points .files-gen at the published generation, inside the |
| +// admitted writer's fence, when it names that generation's cold predecessor |
| +// for this incarnation: the files the start check accepted for this |
| +// session, which the session has served since. Absent, or anything else, is |
| +// left alone for the start check to decide. |
| +func (r movesRecoveryOwners) refreshFilesGen(ctx context.Context, host string, h *hostclaim.Holder, work hostwrite.Work, id hostwrite.Identity) error { |
| + m := r.m |
| + dir, _ := m.FilesDir(host) |
| + return work.With(ctx, id, func() error { |
| + files, _, err := m.localMarkers(host, filepath.Join(dir, ".none")) |
| + if err != nil || files == nil || files.Incarnation != h.Incarnation || files.Op == h.Generation { |
| + return err |
| + } |
| + if pred, ok := coldPredecessor(h, h.Generation); !ok || pred == "" || files.Op != pred { |
| + return nil |
| + } |
| + body, err := json.Marshal(moves.FilesGen{Op: h.Generation, Incarnation: h.Incarnation}) |
| + if err != nil { |
| + return err |
| + } |
| + if err := syncPath(dir); err != nil { |
| + return err |
| + } |
| + return writeFileAtomic(filepath.Join(dir, archive.MarkerName), body, 0o644) |
| + }) |
| +} |
| + |
| +// PublicationEvidence requires this process's receipt for the admitted |
| +// session, as the landing's does; the return generation's marker also |
| +// carries its build's pin and install.json's digest. |
| +func (r movesRecoveryOwners) PublicationEvidence(ctx context.Context, o hostclaim.Observed) error { |
| + v, err := r.recovery(ctx, o) |
| + if err != nil { |
| + return err |
| + } |
| + prepared := "" |
| + if v.dirty { |
| + prepared = v.rb |
| + } |
| + return r.m.publicationEvidence(ctx, o.Host, v.holder, prepared, v.manifest) |
| +} |
diff --git a/internal/host/movesrecoveryowners_test.go b/internal/host/movesrecoveryowners_test.go| new file mode 100644 |
| index 000000000..78a1fe785 |
| --- /dev/null |
| +++ b/internal/host/movesrecoveryowners_test.go |
| @@ -0,0 +1,651 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "os" |
| + "path/filepath" |
| + "reflect" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/activator" |
| + "github.com/carlosframework/platform/internal/genselect" |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| + "github.com/carlosframework/platform/internal/hostwrite" |
| + "github.com/carlosframework/platform/internal/moves" |
| + "github.com/carlosframework/platform/internal/moves/archive" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/superfly/ltx" |
| +) |
| + |
| +// The recovery wake owners (moves Plan 4 Task 9; ruling Q7). The fixture is |
| +// the landing probe's (newPOFixture) turned into the source's recovery: the |
| +// source ie-box executes the move's restoring op at probing (a rollback, |
| +// dirty or clean) or its reverting op at probing (a revert), and the row is |
| +// held by the rollback id or the move id. |
| + |
| +const ( |
| + rcDirty = "dirty" |
| + rcClean = "clean" |
| + rcRevert = "revert" |
| + // rcS is the source installation's stamp S the flip (or the revert) |
| + // journalled. |
| + rcS = 15 |
| +) |
| + |
| +// rcFiles is a fake C8 files manifest provider (Task 6 builds the real |
| +// one): it records the call and runs under the start's work. |
| +type rcFiles struct { |
| + f *poFixture |
| + err error |
| + calls int |
| +} |
| + |
| +func (r *rcFiles) Establish(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { |
| + r.calls++ |
| + r.f.order = append(r.f.order, "establish:"+o.Claim.Holder.Generation) |
| + if r.err != nil { |
| + return r.err |
| + } |
| + return work.With(ctx, in.Identity, func() error { return nil }) |
| +} |
| + |
| +type rcFixture struct { |
| + *poFixture |
| + kind string |
| + c8 *rcFiles |
| + inputs *provision.MoveInputs |
| + manifest moves.InstallManifest // the dirty return's |
| + seal moves.Seal // the return seal |
| +} |
| + |
| +// rcInputs are the inputs the recovery probe recorded: neither the move's |
| +// pins nor the landing's. |
| +func rcInputs(env []string) *provision.MoveInputs { |
| + return &provision.MoveInputs{BundleETags: map[string]string{"default": "e-recovered"}, OverlayETag: "e-overlay-2", Release: "v1.3.0", EnvReport: envDigest(poEnvKey, env)} |
| +} |
| + |
| +func newRCFixture(t *testing.T, kind string) *rcFixture { |
| + t.Helper() |
| + po := newPOFixture(t) |
| + f := &rcFixture{poFixture: po, kind: kind, c8: &rcFiles{f: po}} |
| + f.owners.Box, f.owners.Region = "ie-box", "ie" |
| + f.owners.Files = f.c8 |
| + f.launch.Version = "v1.3.0" |
| + f.inputs = rcInputs(f.launch.Env) |
| + f.renderETags, f.renderOverlay = f.inputs.BundleETags, f.inputs.OverlayETag |
| + at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) |
| + |
| + // The source's own generation in its region, and the marker naming it. |
| + srcKey := "instances/" + poHost + "/g-src/0009/" + ltx.FormatFilename(1, 30) |
| + f.bucket.Put("data", srcKey, f.ltxHeader(1, 30)) |
| + f.bucket.Put("data", "instances/"+poHost+"/generation.json", mustJSON(t, genselect.Marker{Op: "src", Incarnation: f.inc, Host: poHost, Box: "ie-box", InstalledAt: at})) |
| + |
| + c := f.recoveryClaim() |
| + f.rec.Region = "ie" |
| + f.rec.Move.LandingPhase = provision.LandRestored |
| + f.route.Status, f.route.HoldOp = "restoring", rpRB |
| + switch kind { |
| + case rcRevert: |
| + f.rec.Move.State, f.rec.Move.LandingPhase = provision.MoveReverting, "" |
| + f.rec.Move.RecoveryInputs = f.inputs |
| + f.route.HoldOp = poOp |
| + default: |
| + f.rec.Move.State = provision.MoveRollingBack |
| + f.rec.Move.Rollback = &provision.MoveRollback{ID: rpRB, Dirty: kind == rcDirty, TargetPhase: provision.RbTargetReleased, SourcePhase: provision.RbSourceProbing, RecoveryInputs: f.inputs} |
| + } |
| + if kind == rcDirty { |
| + // The return generation the source's install built from the return |
| + // lane, and its manifest in the rollback's journal (installGeneration). |
| + f.rec.Move.Rollback.TargetPhase = provision.RbTargetReturned |
| + key := "instances/" + poHost + "/g-" + rpRB + "/0009/" + ltx.FormatFilename(1, 50) |
| + version := f.bucket.Put("data", key, f.ltxHeader(1, 50)) |
| + f.manifest = moves.InstallManifest{V: 1, Op: rpRB, Host: poHost, N: 50, SnapshotKey: key, SnapshotVersion: version, SnapshotCRC64: 9, DBSHA256: strings.Repeat("e", 64)} |
| + f.bucket.Put("data", moves.InstallManifestKey(moves.GenerationPrefix("instances/"+poHost, rpRB)), mustJSON(t, f.manifest)) |
| + j, err := moves.ReadJournal(f.dataDir, poHost, rpRB) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + body, _ := json.Marshal(f.manifest) |
| + j.Mark(landStepInstall, string(body)) |
| + if err := j.Commit(f.dataDir); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.seal = moves.Seal{V: moves.SealVersion, N: 50, FromDB: true, SrcCovered: true, SealedAt: at, LeaseUntil: at.Add(time.Minute), |
| + Replica: []moves.SealedObject{{Key: "moves/" + poHost + "/" + rpRB + "/replica/0009/" + ltx.FormatFilename(1, 50), VersionID: "r1", Size: int64(ltx.HeaderSize)}}, |
| + Files: []moves.SealedObject{{Key: "moves/" + poHost + "/" + rpRB + "/files/archive.sealed", VersionID: "r2", Size: 10}, {Key: "moves/" + poHost + "/" + rpRB + "/files/archive.json", VersionID: "r3", Size: 10}}} |
| + c.Op.Journal[moves.JournalReturnSeal] = f.seal.Encode() |
| + f.writeLocal(rpRB, rpRB, f.inc) |
| + } else { |
| + // The source's own files and warm database: no moves files marker. |
| + if err := os.Remove(filepath.Join(f.files, archive.MarkerName)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := hostbox.WriteGenerationMarker(f.route.DBPath+".gen", "src"); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if err := c.Validate(); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.putClaim(c) |
| + return f |
| +} |
| + |
| +// recoveryClaim is the claim the recovery runs under. A return: restoring at |
| +// probing, the source executing again at S (ReturnToSource); the holder is |
| +// still the target's session 20 in the landing's generation, or src_gen for |
| +// a clean return. A revert: reverting at rev_phase probing, the holder never |
| +// left the source. |
| +func (f *rcFixture) recoveryClaim() *hostclaim.Claim { |
| + h := &hostclaim.Holder{Family: "instance", Key: "apps/bab/jam/instances/" + poHost + ".json", Account: "bab", App: "jam", CreatedAt: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), Incarnation: f.inc, Location: "instances/" + poHost, |
| + Generation: poOp, Stamp: rcS, Session: 20, WriterBox: "uk-box", |
| + Admissions: []hostclaim.AdmissionRecord{{Session: 15, Box: "ie-box", Generation: "src", Region: "ie", Mode: "cold"}, |
| + {Session: 20, Box: "uk-box", Generation: poOp, Predecessor: "src", Region: "uk", PredecessorRegion: "ie", Mode: "cold", Coverage: "sealed"}}} |
| + journal := map[string]string{hostclaim.JournalSourceStamp: "15", hostclaim.JournalSrcGen: "src", hostclaim.JournalCutSession: "15", hostclaim.JournalEvidenceSource: "moves/bab/jam/" + poHost + "/" + poOp + "/stopped-source.json"} |
| + op := hostclaim.Op{Kind: hostclaim.KindRestoring, ID: poOp, Phase: hostclaim.PhaseProbing, Exec: "ie-box", Participants: []string{"ie-box", "uk-box"}, |
| + Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box", "rollback": rpRB}, Journal: journal} |
| + gen := uint64(30) |
| + switch f.kind { |
| + case rcClean: |
| + h.Generation = "src" |
| + case rcRevert: |
| + h.Generation, h.Session, h.WriterBox, h.Admissions = "src", 15, "ie-box", h.Admissions[:1] |
| + op.Kind = hostclaim.KindReverting |
| + delete(op.Intent, "rollback") |
| + journal[hostclaim.JournalRevPhase] = hostclaim.PhaseProbing |
| + gen = 18 |
| + } |
| + return &hostclaim.Claim{V: 2, Host: poHost, Gen: gen, Holder: h, Op: op} |
| +} |
| + |
| +// policy is the recovery probe adapter's start policy: a probe carrying |
| +// exactly the recorded inputs. |
| +func (f *rcFixture) policy() activator.StartPolicy { |
| + in := *f.inputs |
| + return activator.StartPolicy{MaxStarts: 3, Window: 30 * time.Second, Probe: true, Inputs: &in} |
| +} |
| + |
| +func (f *rcFixture) selectAdmission(o hostclaim.Observed) ClaimLaunchInputs { |
| + f.t.Helper() |
| + in, err := f.owners.Select(f.t.Context(), f.route, o, f.policy()) |
| + if err != nil { |
| + f.t.Fatal(err) |
| + } |
| + return in |
| +} |
| + |
| +// rcMutations are admissions Select never makes; AdmissionEvidence refuses |
| +// every one. |
| +var rcMutations = map[string]func(*hostclaim.Admission, *genselect.Plan){ |
| + "stamp kept": func(a *hostclaim.Admission, _ *genselect.Plan) { a.KeepStamp = true }, |
| + "another region": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Region = "uk" }, |
| + "another box": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Box = "uk-box" }, |
| + "a stale baseline": func(a *hostclaim.Admission, p *genselect.Plan) { p.MaxTXID, a.RemoteMaxTXID = p.MaxTXID-1, p.MaxTXID-1 }, |
| + "another session": func(a *hostclaim.Admission, _ *genselect.Plan) { a.ObservedSession = 14 }, |
| + "warm": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Mode = "warm" }, |
| + "a loss id": func(a *hostclaim.Admission, _ *genselect.Plan) { a.LossID = "loss" }, |
| + "writes lost": func(a *hostclaim.Admission, _ *genselect.Plan) { |
| + a.AdditionalWritesLost = []hostclaim.WritesLost{{Box: "x"}} |
| + }, |
| + "certified": func(a *hostclaim.Admission, _ *genselect.Plan) { a.PredecessorCertified = true }, |
| + "another phase": func(a *hostclaim.Admission, _ *genselect.Plan) { a.ObservedPhase = hostclaim.PhaseClaimed }, |
| + "another op": func(a *hostclaim.Admission, _ *genselect.Plan) { a.ObservedOpID = "mv_other" }, |
| + "the move id as gen": func(a *hostclaim.Admission, _ *genselect.Plan) { a.NewGeneration = poOp }, |
| + "a remote max moved": func(a *hostclaim.Admission, _ *genselect.Plan) { a.RemoteMaxTXID++ }, |
| + "another coverage": func(a *hostclaim.Admission, _ *genselect.Plan) { a.Coverage = "bogus" }, |
| + "another predecessor": func(a *hostclaim.Admission, _ *genselect.Plan) { a.PredecessorRegion = "de" }, |
| +} |
| + |
| +func (f *rcFixture) refusesMutations(o hostclaim.Observed, in ClaimLaunchInputs) { |
| + f.t.Helper() |
| + for name, mutate := range rcMutations { |
| + a, p := in.Admission, *in.Baseline |
| + mutate(&a, &p) |
| + if err := f.owners.AdmissionEvidence(f.t.Context(), o, a, &p); !errors.Is(err, hostbox.ErrChanged) { |
| + f.t.Fatalf("%s: %v", name, err) |
| + } |
| + } |
| +} |
| + |
| +// Plan 4 Task 9: a dirty return's probe is a cold admission into the return |
| +// generation (the rollback id), sealed, from the return generation's pin at |
| +// the return seal's maximum, and it keeps no stamp: the admission moves |
| +// holder.stamp from S to the new session and holder.generation to the |
| +// return generation, so S is retired (a selection captured at S is refused). |
| +func TestDirtyReturnProbeIssuesANewStamp(t *testing.T) { |
| + f := newRCFixture(t, rcDirty) |
| + ctx := t.Context() |
| + o := f.read() |
| + in := f.selectAdmission(o) |
| + want := hostclaim.Admission{ObservedOpID: poOp, ObservedPhase: hostclaim.PhaseProbing, ObservedSession: 20, ObservedGeneration: poOp, Box: "ie-box", Mode: "cold", |
| + NewGeneration: rpRB, Coverage: "sealed", Region: "ie", PredecessorRegion: "uk", RemoteMaxTXID: 50} |
| + if !reflect.DeepEqual(in.Admission, want) { |
| + t.Fatalf("admission:\n got %+v\nwant %+v", in.Admission, want) |
| + } |
| + if !in.Prepared || in.Baseline == nil || in.Baseline.Generation != rpRB || in.Baseline.MaxTXID != f.seal.N || in.Baseline.Pin == nil || in.Baseline.Pin.SnapshotVersion != f.manifest.SnapshotVersion { |
| + t.Fatalf("baseline: prepared=%v %+v", in.Prepared, in.Baseline) |
| + } |
| + if in.Identity != mpLaunchIdentity(f.inputs) || in.Version != "v1.3.0" || in.DBPath != f.route.DBPath || in.ReadyPath != "/healthz" || !contains(in.Env, "CARLOS_MOVE_PROBE=1") { |
| + t.Fatalf("launch inputs: %+v", in) |
| + } |
| + if err := f.owners.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.refusesMutations(o, in) |
| + unsealed := in.Admission |
| + unsealed.Coverage = "" |
| + if err := f.owners.AdmissionEvidence(ctx, o, unsealed, in.Baseline); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("an unsealed dirty admission: %v", err) |
| + } |
| + |
| + admitted := f.admit(o, in) |
| + h := admitted.Claim.Holder |
| + last := h.Admissions[len(h.Admissions)-1] |
| + if h.Stamp == rcS || h.Stamp != h.Session || h.Generation != rpRB || h.WriterBox != "ie-box" || last.Coverage != "sealed" || last.WritesLost != nil || last.Predecessor != poOp { |
| + t.Fatalf("the admission: holder %+v, last %+v", h, last) |
| + } |
| + policy := MovesClaimPolicy(f.owners.Records) |
| + s := hostbox.Selection{Host: poHost, Action: movesActionSpawn, Box: "ie-box", Region: "ie", Kind: hostclaim.KindRestoring, Op: poOp, Phase: hostclaim.PhaseProbing, |
| + Captured: hostguard.Captured{Incarnation: f.inc, Stamp: rcS}} |
| + rpRefused(t, policy(ctx, admitted.Claim, s, in.Identity), "the capture is not the holder's") |
| + s.Captured.Stamp = h.Stamp |
| + if err := policy(ctx, admitted.Claim, s, in.Identity); err != nil { |
| + t.Fatalf("the new stamp's spawn: %v", err) |
| + } |
| + |
| + // A restarted attempt over the admitted return generation is a fresh |
| + // generation with no coverage, restored from the return generation's |
| + // pin while it has published nothing. |
| + again, err := f.owners.Select(ctx, f.route, admitted, f.policy()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if a := again.Admission; a.NewGeneration != "fresh1" || a.KeepStamp || a.Coverage != "" || a.ObservedGeneration != rpRB || a.PredecessorRegion != "ie" || again.Prepared || |
| + again.Baseline == nil || again.Baseline.Generation != rpRB || again.Baseline.MaxTXID != 50 { |
| + t.Fatalf("restarted: %+v prepared=%v baseline %+v", a, again.Prepared, again.Baseline) |
| + } |
| + if err := f.owners.AdmissionEvidence(ctx, admitted, again.Admission, again.Baseline); err != nil { |
| + t.Fatal(err) |
| + } |
| + reused := again.Admission |
| + reused.NewGeneration = rpRB |
| + if err := f.owners.AdmissionEvidence(ctx, admitted, reused, again.Baseline); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("the return generation twice: %v", err) |
| + } |
| +} |
| + |
| +// Plan 4 Task 9: a clean return's probe is a cold admission into a fresh |
| +// generation restored from src_gen in this region, its coverage |
| +// clean-return: the admission records no writes lost. |
| +func TestCleanReturnProbeIsCoveredCleanReturn(t *testing.T) { |
| + f := newRCFixture(t, rcClean) |
| + ctx := t.Context() |
| + o := f.read() |
| + in := f.selectAdmission(o) |
| + want := hostclaim.Admission{ObservedOpID: poOp, ObservedPhase: hostclaim.PhaseProbing, ObservedSession: 20, ObservedGeneration: "src", Box: "ie-box", Mode: "cold", |
| + NewGeneration: "fresh1", Coverage: "clean-return", Region: "ie", PredecessorRegion: "uk", RemoteMaxTXID: 30} |
| + if !reflect.DeepEqual(in.Admission, want) { |
| + t.Fatalf("admission:\n got %+v\nwant %+v", in.Admission, want) |
| + } |
| + if in.Prepared || in.Baseline == nil || in.Baseline.Generation != "src" || in.Baseline.MaxTXID != 30 { |
| + t.Fatalf("baseline: prepared=%v %+v", in.Prepared, in.Baseline) |
| + } |
| + if err := f.owners.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.refusesMutations(o, in) |
| + for _, coverage := range []string{"", "sealed"} { |
| + a := in.Admission |
| + a.Coverage = coverage |
| + if err := f.owners.AdmissionEvidence(ctx, o, a, in.Baseline); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("coverage %q: %v", coverage, err) |
| + } |
| + } |
| + admitted := f.admit(o, in) |
| + h := admitted.Claim.Holder |
| + last := h.Admissions[len(h.Admissions)-1] |
| + if last.Coverage != "clean-return" || last.WritesLost != nil || len(last.AdditionalWritesLost) != 0 || h.Generation != "fresh1" || h.Stamp == rcS { |
| + t.Fatalf("the admission: holder %+v, last %+v", h, last) |
| + } |
| + |
| + // A clean return whose holder does not name src_gen is not this return. |
| + t.Run("the holder is not at src_gen", func(t *testing.T) { |
| + f := newRCFixture(t, rcClean) |
| + c := f.recoveryClaim() |
| + c.Holder.Generation = poOp |
| + f.putClaim(c) |
| + if _, err := f.owners.Select(t.Context(), f.route, f.read(), f.policy()); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatal(err) |
| + } |
| + }) |
| + // A declared or fenced return (Q6) asks no coverage: the claim journals |
| + // the fence and the record carries it as its loss. |
| + t.Run("a fenced return asks no coverage", func(t *testing.T) { |
| + f := newRCFixture(t, rcClean) |
| + c := f.recoveryClaim() |
| + c.Op.Journal["fence.uk-box"] = `{"key":"fences/uk-box"}` |
| + f.putClaim(c) |
| + f.rec.Move.Rollback.LossRef = "fences/uk-box" |
| + o := f.read() |
| + in := f.selectAdmission(o) |
| + if in.Admission.Coverage != "" || in.Admission.NewGeneration != "fresh1" { |
| + t.Fatalf("admission %+v", in.Admission) |
| + } |
| + if err := f.owners.AdmissionEvidence(t.Context(), o, in.Admission, in.Baseline); err != nil { |
| + t.Fatal(err) |
| + } |
| + covered := in.Admission |
| + covered.Coverage = "clean-return" |
| + if err := f.owners.AdmissionEvidence(t.Context(), o, covered, in.Baseline); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatal(err) |
| + } |
| + // The record and the claim disagree: wait. |
| + f.rec.Move.Rollback.LossRef = "" |
| + if _, err := f.owners.Select(t.Context(), f.route, o, f.policy()); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatal(err) |
| + } |
| + }) |
| +} |
| + |
| +// A revert's probe (ruling Q7): a cold admission into a fresh generation |
| +// restored from the holder's own generation, which a revert never changed, |
| +// with no coverage: the admission records the source's writes as possibly |
| +// lost (uncovered), and keeps no stamp. |
| +func TestRevertProbeAdmitsUncovered(t *testing.T) { |
| + f := newRCFixture(t, rcRevert) |
| + ctx := t.Context() |
| + o := f.read() |
| + in := f.selectAdmission(o) |
| + want := hostclaim.Admission{ObservedOpID: poOp, ObservedPhase: hostclaim.PhaseProbing, ObservedSession: 15, ObservedGeneration: "src", Box: "ie-box", Mode: "cold", |
| + NewGeneration: "fresh1", Region: "ie", PredecessorRegion: "ie", RemoteMaxTXID: 30} |
| + if !reflect.DeepEqual(in.Admission, want) { |
| + t.Fatalf("admission:\n got %+v\nwant %+v", in.Admission, want) |
| + } |
| + if in.Prepared || in.Baseline == nil || in.Baseline.Generation != "src" || in.Identity != mpLaunchIdentity(f.inputs) { |
| + t.Fatalf("inputs: %+v", in) |
| + } |
| + if err := f.owners.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil { |
| + t.Fatal(err) |
| + } |
| + f.refusesMutations(o, in) |
| + for _, coverage := range []string{"clean-return", "sealed"} { |
| + a := in.Admission |
| + a.Coverage = coverage |
| + if err := f.owners.AdmissionEvidence(ctx, o, a, in.Baseline); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("coverage %q: %v", coverage, err) |
| + } |
| + } |
| + admitted := f.admit(o, in) |
| + h := admitted.Claim.Holder |
| + last := h.Admissions[len(h.Admissions)-1] |
| + if last.Coverage != "" || last.WritesLost == nil || last.WritesLost.Box != "ie-box" || last.WritesLost.Session != 15 || last.WritesLost.Generation != "src" || last.WritesLost.RemoteMaxTXID != 30 || |
| + h.Generation != "fresh1" || h.Stamp == rcS { |
| + t.Fatalf("the admission: holder %+v, last %+v", h, last) |
| + } |
| + |
| + // The revert's inputs are exactly the recorded ones, and its row is held |
| + // by the move id. |
| + for name, mutate := range map[string]func(*rcFixture, *activator.StartPolicy){ |
| + "other inputs": func(_ *rcFixture, p *activator.StartPolicy) { p.Inputs.Release = "v9" }, |
| + "no inputs": func(_ *rcFixture, p *activator.StartPolicy) { p.Inputs = nil }, |
| + "nothing recorded": func(f *rcFixture, _ *activator.StartPolicy) { f.rec.Move.RecoveryInputs = nil }, |
| + "not a probe": func(_ *rcFixture, p *activator.StartPolicy) { p.Probe = false }, |
| + "a rollback's hold": func(f *rcFixture, _ *activator.StartPolicy) { f.route.HoldOp = rpRB }, |
| + "reopened": func(f *rcFixture, _ *activator.StartPolicy) { now := time.Now(); f.rec.Move.ReopenedAt = &now }, |
| + "another live release": func(f *rcFixture, _ *activator.StartPolicy) { f.launch.Version = "v1.2.3" }, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + f := newRCFixture(t, rcRevert) |
| + p := f.policy() |
| + mutate(f, &p) |
| + if _, err := f.owners.Select(t.Context(), f.route, f.read(), p); err == nil { |
| + t.Fatal("selected") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// Ruling Q7: the moves owner set dispatches by the observed op's kind. A |
| +// landing reaches the landing probe's owners, a restoring or reverting op |
| +// the recovery owners, and the moving kind nothing; without C8 the |
| +// recovery kinds refuse, typed, while the landing is unchanged. |
| +func TestMovesOwnersDispatchByKind(t *testing.T) { |
| + ctx := t.Context() |
| + dispatch := func(m *movesProbeOwners) ClaimOwnerDispatch { |
| + owners := m.MovesOwners(&movesProbeAttempts{}).ClaimKindOwners |
| + return ClaimOwnerDispatch{Moves: &owners} |
| + } |
| + |
| + landing := newPOFixture(t) |
| + landing.owners.Files = &rcFiles{f: landing} |
| + in, err := dispatch(landing.owners).Select(ctx, landing.route, landing.read(), poProbe) |
| + if err != nil || in.Admission.NewGeneration != poOp || in.Admission.Region != "uk" { |
| + t.Fatalf("landing: %+v %v", in.Admission, err) |
| + } |
| + |
| + for _, kind := range []string{rcDirty, rcClean, rcRevert} { |
| + f := newRCFixture(t, kind) |
| + d := dispatch(f.owners) |
| + o := f.read() |
| + in, err := d.Select(ctx, f.route, o, f.policy()) |
| + if err != nil { |
| + t.Fatalf("%s: %v", kind, err) |
| + } |
| + if in.Admission.Region != "ie" || in.Admission.Box != "ie-box" || (kind == rcDirty) != (in.Admission.NewGeneration == rpRB) { |
| + t.Fatalf("%s: %+v", kind, in.Admission) |
| + } |
| + if err := d.AdmissionEvidence(ctx, o, in.Admission, in.Baseline); err != nil { |
| + t.Fatalf("%s evidence: %v", kind, err) |
| + } |
| + if err := d.EmptyAuthority(ctx, o, in); !errors.Is(err, hostbox.ErrUnsupported) { |
| + t.Fatalf("%s empty authority: %v", kind, err) |
| + } |
| + |
| + // Without C8 the recovery refuses before reading anything. |
| + f.owners.Files = nil |
| + f.launches = 0 |
| + _, err = d.Select(ctx, f.route, o, f.policy()) |
| + var missing *ClaimDependencyError |
| + if !errors.As(err, &missing) || !strings.Contains(missing.Error(), "C8") || f.launches != 0 { |
| + t.Fatalf("%s without C8: %v", kind, err) |
| + } |
| + } |
| + landing.owners.Files = nil |
| + if _, err := dispatch(landing.owners).Select(ctx, landing.route, landing.read(), poProbe); err != nil { |
| + t.Fatalf("the landing without C8: %v", err) |
| + } |
| + |
| + // The moving kind reaches no owner. |
| + moving := newPOFixture(t) |
| + c := moving.poFlipClaim() |
| + c.Op.Kind, c.Op.Phase = hostclaim.KindMoving, hostclaim.PhaseSealed |
| + moving.putClaim(c) |
| + if _, err := dispatch(moving.owners).Select(ctx, moving.route, moving.read(), poProbe); !errors.Is(err, hostbox.ErrUnsupported) { |
| + t.Fatalf("moving: %v", err) |
| + } |
| +} |
| + |
| +// Ruling Q7's Materialize: before every start of the admitted recovery |
| +// probe, the held render and the held release of the recorded inputs, each |
| +// under its own held work at the admitted capture, then the return |
| +// generation's verified marker (dirty), then C8's Establish under the |
| +// start's work, then the start check. A changed render, a refused held |
| +// work or a refused Establish stops the start. |
| +func TestRecoveryMaterializeRendersTheRecordedInputsUnderHeldWork(t *testing.T) { |
| + ctx := t.Context() |
| + start := func(t *testing.T, kind string) (*rcFixture, hostclaim.Observed, ClaimLaunchInputs) { |
| + f := newRCFixture(t, kind) |
| + o := f.read() |
| + in := f.selectAdmission(o) |
| + admitted := f.admit(o, in) |
| + if kind != rcDirty { |
| + // The restore rewrites <dbPath>.gen to the admitted generation. |
| + if err := hostbox.WriteGenerationMarker(f.route.DBPath+".gen", admitted.Claim.Holder.Generation); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + f.order = nil |
| + return f, admitted, in |
| + } |
| + cfg := func(f *rcFixture) string { |
| + return HeldConfigIdentity(f.inputs.BundleETags, f.inputs.OverlayETag).Config |
| + } |
| + launch := func(f *rcFixture) string { id := mpLaunchIdentity(f.inputs); return id.Config + "|" + id.Release } |
| + |
| + t.Run("dirty", func(t *testing.T) { |
| + f, o, in := start(t, rcDirty) |
| + if err := f.owners.Materialize(ctx, o, in, poWork{action: "spawn", f: f.poFixture}); err != nil { |
| + t.Fatal(err) |
| + } |
| + // The admitted writer's fence first, then the held render and release |
| + // each under its own selection, then C8 under the start's work. |
| + rcInOrder(t, f.order, "spawn:"+launch(f), "render", "select:held-config", "held-config:"+cfg(f)+"|", "select:held-release", "release:v1.3.0", "held-release:|v1.3.0", |
| + "establish:"+rpRB, "spawn:"+launch(f)) |
| + if f.c8.calls != 1 { |
| + t.Fatalf("C8 ran %d times", f.c8.calls) |
| + } |
| + // The verified marker names the return generation with its pin. |
| + marker, err := f.owners.readMarker(ctx, o.Claim.Holder) |
| + if err != nil || marker == nil || marker.Op != rpRB || marker.Pin != manifestPin(f.manifest) || marker.Box != "ie-box" { |
| + t.Fatalf("marker %+v %v", marker, err) |
| + } |
| + if err := f.owners.Publish(ctx, o, in); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := f.owners.PublicationEvidence(ctx, o); err != nil { |
| + t.Fatalf("the return generation's publication: %v", err) |
| + } |
| + }) |
| + t.Run("clean", func(t *testing.T) { |
| + f, o, in := start(t, rcClean) |
| + if err := f.owners.Materialize(ctx, o, in, poWork{action: "spawn", f: f.poFixture}); err != nil { |
| + t.Fatal(err) |
| + } |
| + rcInOrder(t, f.order, "render", "held-config:"+cfg(f)+"|", "release:v1.3.0", "held-release:|v1.3.0", "establish:fresh1") |
| + if f.c8.calls != 1 { |
| + t.Fatalf("C8 ran %d times", f.c8.calls) |
| + } |
| + // No prepared generation: the marker still names src_gen until the |
| + // fenced publication. |
| + if marker, err := f.owners.readMarker(ctx, o.Claim.Holder); err != nil || marker.Op != "src" { |
| + t.Fatalf("marker %+v %v", marker, err) |
| + } |
| + if err := f.owners.Publish(ctx, o, in); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatalf("an unfenced publication of a fresh generation: %v", err) |
| + } |
| + // The fenced publication after the session's own first root. |
| + f.bucket.Put("data", "instances/"+poHost+"/g-fresh1/0009/"+ltx.FormatFilename(1, 31), f.ltxHeader(1, 31)) |
| + checks := 0 |
| + pub := ClaimPublication{Sync: func(context.Context) error { return nil }, Check: func(context.Context) error { checks++; return nil }} |
| + receipt, err := f.owners.PublishFenced(ctx, o, in, pub) |
| + if err != nil || receipt.MarkerVersion == "" || checks == 0 { |
| + t.Fatalf("PublishFenced: %+v %v (checks %d)", receipt, err, checks) |
| + } |
| + if marker, err := f.owners.readMarker(ctx, o.Claim.Holder); err != nil || marker.Op != "fresh1" || marker.Box != "ie-box" { |
| + t.Fatalf("marker %+v %v", marker, err) |
| + } |
| + if err := f.owners.PublicationEvidence(ctx, o); err != nil { |
| + t.Fatalf("the fresh generation's publication: %v", err) |
| + } |
| + }) |
| + t.Run("the render is not the recorded inputs", func(t *testing.T) { |
| + f, o, in := start(t, rcRevert) |
| + f.renderETags = map[string]string{"default": "e-moved-on"} |
| + if err := f.owners.Materialize(ctx, o, in, poWork{action: "spawn", f: f.poFixture}); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatal(err) |
| + } |
| + if contains(f.order, "select:held-config") || f.c8.calls != 0 { |
| + t.Fatalf("selected or established after a changed render: %q", f.order) |
| + } |
| + }) |
| + t.Run("the launch environment is not the recorded one", func(t *testing.T) { |
| + f, o, in := start(t, rcClean) |
| + in.Env = append(in.Env, "B=2") |
| + if err := f.owners.Materialize(ctx, o, in, poWork{action: "spawn", f: f.poFixture}); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatal(err) |
| + } |
| + if f.c8.calls != 0 { |
| + t.Fatal("established") |
| + } |
| + }) |
| + t.Run("another identity", func(t *testing.T) { |
| + f, o, in := start(t, rcClean) |
| + in.Identity = mpLaunchIdentity(mpPins()) |
| + if err := f.owners.Materialize(ctx, o, in, poWork{action: "spawn", f: f.poFixture}); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatal(err) |
| + } |
| + if len(f.order) != 0 { |
| + t.Fatalf("ran %q", f.order) |
| + } |
| + }) |
| + t.Run("the start's work is refused", func(t *testing.T) { |
| + f, o, in := start(t, rcDirty) |
| + if err := f.owners.Materialize(ctx, o, in, poWork{action: "spawn", f: f.poFixture, shut: true}); !errors.Is(err, hostwrite.ErrUnavailable) { |
| + t.Fatal(err) |
| + } |
| + if contains(f.order, "render") || f.c8.calls != 0 { |
| + t.Fatalf("ran %q", f.order) |
| + } |
| + }) |
| + t.Run("C8 refuses", func(t *testing.T) { |
| + f, o, in := start(t, rcClean) |
| + f.c8.err = errors.New("the bucket cannot be reached") |
| + if err := f.owners.Materialize(ctx, o, in, poWork{action: "spawn", f: f.poFixture}); !errors.Is(err, f.c8.err) { |
| + t.Fatal(err) |
| + } |
| + }) |
| + t.Run("not the admitted session", func(t *testing.T) { |
| + f := newRCFixture(t, rcClean) |
| + o := f.read() |
| + in := f.selectAdmission(o) |
| + if err := f.owners.Materialize(ctx, o, in, poWork{action: "spawn", f: f.poFixture}); !errors.Is(err, hostbox.ErrChanged) { |
| + t.Fatal(err) |
| + } |
| + if len(f.order) != 0 { |
| + t.Fatalf("ran %q", f.order) |
| + } |
| + }) |
| +} |
| + |
| +// Until Task 4 binds the loss contract (LossSources), an admission while a |
| +// declared loss is pending is refused, typed, by the recovery's |
| +// AdmissionEvidence: the writer substitutes the frozen maximum, which only |
| +// the loss sources can vouch for. |
| +func TestRecoveryAdmissionEvidenceRefusesAPendingLossUntilTheLossContract(t *testing.T) { |
| + f := newRCFixture(t, rcClean) |
| + c := f.recoveryClaim() |
| + c.LossRefs = []string{"loss-1"} |
| + c.Op.Journal["loss.loss-1"] = `{"kind":"move-target-lost","target":"uk-box"}` |
| + f.putClaim(c) |
| + f.rec.Move.Rollback.LossRef = "losses/loss-1" |
| + o := f.read() |
| + in := f.selectAdmission(o) |
| + if in.Admission.Coverage != "" { |
| + t.Fatalf("a declared return asked coverage %q", in.Admission.Coverage) |
| + } |
| + err := f.owners.AdmissionEvidence(t.Context(), o, in.Admission, in.Baseline) |
| + var missing *ClaimDependencyError |
| + if !errors.As(err, &missing) { |
| + t.Fatalf("not the typed refusal: %v", err) |
| + } |
| +} |
| + |
| +// rcInOrder requires each of want in order, as a subsequence of got. |
| +func rcInOrder(t *testing.T, got []string, want ...string) { |
| + t.Helper() |
| + i := 0 |
| + for _, g := range got { |
| + if i < len(want) && g == want[i] { |
| + i++ |
| + } |
| + } |
| + if i != len(want) { |
| + t.Fatalf("order:\n got %q\nwant, in order, %q (missing from %q)", got, want, want[i]) |
| + } |
| +} |