amadan
host: one generation installer for the landing and the dirty return (§7.6, §7.8)
Plan 4 Task 3 with rulings J13, J14, J20 and J24. The landing's parking, building, published, local and configured move into installGeneration (movesinstall.go), driven by an installSource that carries what differs between the two kinds of install: - the claim check (landing op at the phase, mirrored; restoring/claimed with this box executing as the source at S, for this rollback) and the record check (landing with no reason; rolling_back, dirty, the source at the phase), used by each phase and by the host guard; - the refusal action: the landing's MoveMarkRollback, the return's MoveMarkStuck with StuckReturnFilesOccupied or StuckReturnInstallMismatch; - the phase advance as a caller-bound closure over an installStamp (MoveLandingPhase stamping installed_db_sha256, MoveRollbackSourcePhase stamping RecoveryInputs), so neither binds through a type switch; - the generation (move id, or the rollback id: J14, cold, never S's name), which names the build, the install's journal, .gen and .files-gen; - configured: a return renders current inputs and records them in Rollback.RecoveryInputs at the CAS that ends configured (G8), never compared with move.inputs; - local on the source: RetireArtifacts(S) when the inventory lists paths of S, then the H5(b) moves-owned rename (parkLocalArtifacts) of every pre-move artifact still at the flat paths, before any extraction; so placeStaged and restore never install over the source's own files; - the stand-in park view takes a role: From authoritative for a landing, To for a return (J24). Every seal object must sit in the install's lane. The Plan 2 landing suite passes unchanged. recoveryInstallSource is the return's binding for Task 5; nothing in production calls it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
7 files changed,
+1882
−643
approved_moves_copy_test.go+1 −0internal/host/movesfinal.go+5 −5internal/host/movesinstall.go+981 −0internal/host/movesinstall_test.go+667 −0internal/host/movesland.go+64 −510internal/host/moveslandinstall.go+81 −79internal/host/moveslandpark.go+83 −49
diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go| index 373391082..828b8c2c9 100644 |
| --- a/approved_moves_copy_test.go |
| +++ b/approved_moves_copy_test.go |
| @@ -576,6 +576,7 @@ var movesScanFiles = []string{ |
| "internal/host/movesguard.go", |
| "internal/host/movesheldenv.go", |
| "internal/host/movesheldlock.go", |
| + "internal/host/movesinstall.go", |
| "internal/host/moveshold.go", |
| "internal/host/movesland.go", |
| "internal/host/moveslandinstall.go", |
diff --git a/internal/host/movesfinal.go b/internal/host/movesfinal.go| index e02647890..c4492f175 100644 |
| --- a/internal/host/movesfinal.go |
| +++ b/internal/host/movesfinal.go |
| @@ -471,12 +471,12 @@ func (f *sourceFinal) installation(ctx context.Context) (string, error) { |
| } |
| // sourceArtifactPaths are the source's artifacts at the flat paths (G11), |
| -// relative to the data directory: the database with its marker, journal, |
| -// quarantine and litestream shadow, the files directory and <host>.env. |
| +// relative to the data directory: the installation's (installArtifactPaths: |
| +// the database with its marker, journal, quarantine and litestream shadow, |
| +// the files directory and <host>.env), then the rendered config and the |
| +// sidecar's env. The order is the journalled plan's and does not change. |
| func sourceArtifactPaths(host string) []string { |
| - db := host + ".db" |
| - return []string{db, db + ".gen", db + "-wal", db + "-shm", db + ".corrupt", ".activator-" + db + "-litestream", host, host + ".env", |
| - host + ".config.env", host + "-sidecar.env"} |
| + return append(installArtifactPaths(host), host+".config.env", host+"-sidecar.env") |
| } |
| // deleteRow is step 3's row and timers delete, under the host's hold lock |
diff --git a/internal/host/movesinstall.go b/internal/host/movesinstall.go| new file mode 100644 |
| index 000000000..6c03b26b5 |
| --- /dev/null |
| +++ b/internal/host/movesinstall.go |
| @@ -0,0 +1,981 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "log" |
| + "path/filepath" |
| + "sort" |
| + "strconv" |
| + "strings" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| + "github.com/carlosframework/platform/internal/lease" |
| + "github.com/carlosframework/platform/internal/moves" |
| + "github.com/carlosframework/platform/internal/moves/archive" |
| + "github.com/carlosframework/platform/internal/park" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// The generation installer (moves spec §7.6, §7.8; Plan 4 Task 3), shared by |
| +// the landing on the target and the dirty return on the source. It runs the |
| +// six phases parking, building, published, local, configured and restored |
| +// exactly as Plan 2 Task 10 wrote them for the landing; installSource says |
| +// what differs between the two kinds of install (rulings J13, J14, J20, J24): |
| +// |
| +// - the claim check: a landing's op is landing at the phase, mirrored into |
| +// the claim by every record step; a return's op is restoring/claimed with |
| +// the source executing at its pre-move stamp S throughout, and the claim |
| +// does not follow the source's install phases; |
| +// - the record check: landing at the phase with no rollback reason, or |
| +// rolling_back, dirty, with the source at the phase; |
| +// - the refusal action: a landing marks its rollback (MoveMarkRollback); a |
| +// return stops for an operator (MoveMarkStuck) with a recorded reason; |
| +// - the phase advance: MoveLandingPhase or MoveRollbackSourcePhase, bound |
| +// by the caller, so neither kind is told apart by a type switch; |
| +// - the generation: the move id for a landing, the rollback id for a return |
| +// (J14: a return is cold and never reuses S's generation name). It names |
| +// the build, the install's own journal, <dbPath>.gen and .files-gen; |
| +// - configured: a landing renders and requires its pinned inputs; a return |
| +// renders the current ones and records them in Rollback.RecoveryInputs, |
| +// never compared with move.inputs (J2-R2-5); |
| +// - local: a landing parks an earlier stay's artifacts tagged S'; a return |
| +// parks the source's own pre-move installation before anything is |
| +// extracted (J13): RetireArtifacts(S) when the claim's inventory lists |
| +// paths of S, then the moves-owned rename (H5(b), parkLocalArtifacts) of |
| +// every pre-move artifact still at the flat paths. placeStaged and |
| +// restore then find nothing there, and refuse anything that appears; |
| +// - the stand-in park view's authoritative region (J24): the source's for |
| +// a landing, the target's for a return, each the other side of the move |
| +// before this side's probe is admitted. |
| +// |
| +// The phase-name convention is Plan 2's (G8): restored is written by the CAS |
| +// that ends configured, after the database is installed and its markers |
| +// written. Every record write goes through a provision transition (the |
| +// record_seq rule). The installer is unarmed in production, as the landing |
| +// is (ruling G17): only the landing's pass and the tests call it. |
| + |
| +// installStepPreMove is the moves-owned rename of a return's pre-move |
| +// artifacts (parkLocalArtifacts' step: the retirement id). |
| +const installStepPreMove = "pre_move_parked" |
| + |
| +// installStamp is what a phase step records besides the phase: at the CAS |
| +// that ends configured, the installed database's digest (a landing records |
| +// it in landing.installed_db_sha256) and the inputs a return rendered. |
| +type installStamp struct { |
| + InstalledDBSHA256 string |
| + RecoveryInputs *provision.MoveInputs |
| +} |
| + |
| +// installRefusal is why an install stops for good: installSource.Refuse maps |
| +// it to its kind's recorded sentence. |
| +type installRefusal int |
| + |
| +const ( |
| + refuseFilesOccupied installRefusal = iota + 1 // local: the files paths hold what the install did not write |
| + refuseInstallMismatch // configured: the build does not verify |
| + refuseConfigChanged // configured: a landing's render is not its pin |
| + refuseDeadline // configured: a landing ran past its deadline |
| +) |
| + |
| +// installSource is one kind of install: what the six phases need from the |
| +// caller (Plan 4 Task 3's interface, with the rulings' additions). |
| +type installSource struct { |
| + OpID, Host string |
| + // Role is roleTarget for a landing, roleSource for a dirty return. |
| + Role string |
| + // Name is the install's word in the log: "landing" or "return". |
| + Name string |
| + // Lane is the staging or return lane prefix in this box's own bucket; |
| + // every object the seal names must be under it. |
| + Lane string |
| + // Seal reads the sealed set from the claim the phase works from. |
| + Seal func(*hostclaim.Claim) (moves.Seal, error) |
| + // Baseline is what the sealed set restores to: the final TXID, the |
| + // files and timers digests, and the carried columns. |
| + Baseline *provision.MoveSource |
| + // GenerationOp names the generation this install builds and the |
| + // install's own journal; HoldOp is the op the row is held by. |
| + GenerationOp, HoldOp string |
| + // PinInputs are a landing's pinned inputs; nil for a return. |
| + PinInputs *provision.MoveInputs |
| + // Phases are the six phase names of this install, in order. |
| + Phases [6]string |
| + // Phase is the record transition one step makes, bound to the guard. |
| + Phase func(from, to string, st installStamp) func(*provision.Record) error |
| + // Mirror is the claim half of a record step; nil when the claim does |
| + // not follow this install's phases. |
| + Mirror func(ctx context.Context, from, to string) error |
| + // Sync is each phase's first claim read (a landing walks a claim that |
| + // is behind forward); ClaimCheck is what the phase and the guard require |
| + // of the claim at phase. |
| + Sync func(ctx context.Context) (hostclaim.Observed, error) |
| + ClaimCheck func(phase string) func(*hostclaim.Claim) error |
| + // RecordPhase answers the phase a record is at while it is still this |
| + // install's, and an error otherwise. |
| + RecordPhase func(*provision.Record) (string, error) |
| + // Refuse writes the refusal: written is true when the record took it, |
| + // sentence is the recorded reason. |
| + Refuse func(ctx context.Context, why installRefusal) (written bool, sentence string, err error) |
| + // PastDeadline reports a landing's deadline passed; nil for a return. |
| + PastDeadline func() bool |
| +} |
| + |
| +func (src installSource) validate() error { |
| + if src.OpID == "" || src.Host == "" || (src.Role != roleTarget && src.Role != roleSource) || src.Lane == "" || src.Seal == nil || |
| + src.GenerationOp == "" || src.HoldOp == "" || src.Phase == nil || src.Sync == nil || src.ClaimCheck == nil || |
| + src.RecordPhase == nil || src.Refuse == nil { |
| + return errors.New("moves: an install source names its op, role, lane, seal, generation and hooks") |
| + } |
| + for _, p := range src.Phases { |
| + if p == "" { |
| + return errors.New("moves: an install source names its six phases") |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// kind is the claim op kind this install runs under. |
| +func (src installSource) kind() string { |
| + if src.Role == roleSource { |
| + return hostclaim.KindRestoring |
| + } |
| + return hostclaim.KindLanding |
| +} |
| + |
| +// installRun is one call's state for one install. |
| +type installRun struct { |
| + w *movesWorker |
| + s release.Store |
| + rec provision.Record |
| + g provision.MoveGuard |
| + src installSource |
| + guard hostguard.Guard |
| + local park.Local |
| + // refused is the sentence a refusal recorded during this call. |
| + refused string |
| +} |
| + |
| +// installGeneration runs the record's current install phase once (one phase |
| +// per call, as the landing's pass and the cut step): parking, building, |
| +// published, local or configured. advanced is true when it wrote the record |
| +// (a phase step or the refusal); refused is the recorded sentence when the |
| +// refusal was written. A record at restored is the probe's and is left as it |
| +// is; one that is no longer this install's is an error. |
| +func installGeneration(ctx context.Context, s release.Store, w *movesWorker, rec provision.Record, src installSource) (advanced bool, refused string, err error) { |
| + if err := src.validate(); err != nil { |
| + return false, "", err |
| + } |
| + phase, err := src.RecordPhase(&rec) |
| + if err != nil { |
| + return false, "", err |
| + } |
| + if rec.Move.ID != src.OpID || rec.Host != src.Host { |
| + return false, "", fmt.Errorf("moves: the install of %s is not %s's", src.OpID, rec.Host) |
| + } |
| + ir := newInstallRun(s, w, rec, src) |
| + switch phase { |
| + case src.Phases[0]: |
| + advanced, err = ir.parking(ctx) |
| + case src.Phases[1]: |
| + advanced, err = ir.building(ctx) |
| + case src.Phases[2]: |
| + advanced, err = ir.published(ctx) |
| + case src.Phases[3]: |
| + advanced, err = ir.local_(ctx) |
| + case src.Phases[4]: |
| + advanced, err = ir.configured(ctx) |
| + case src.Phases[5]: |
| + return false, "", nil |
| + default: |
| + return false, "", fmt.Errorf("moves: %s is not a phase of the %s's install", phase, src.Name) |
| + } |
| + return advanced, ir.refused, err |
| +} |
| + |
| +func newInstallRun(s release.Store, w *movesWorker, rec provision.Record, src installSource) *installRun { |
| + cfg := w.cfg |
| + ir := &installRun{w: w, s: s, rec: rec, g: provision.MoveGuard{ID: rec.Move.ID, Incarnation: rec.Move.Incarnation}, src: src} |
| + ir.guard = cfg.Guard |
| + if ir.guard.Claims == nil { |
| + ir.guard.Claims = hostguard.StoreReader{Store: s} |
| + } |
| + ir.local = cfg.Local |
| + ir.local.Guard, ir.local.Claims = ir.guard, s |
| + if ir.local.Box == "" { |
| + ir.local.Box = cfg.Box |
| + } |
| + if ir.local.Authorize == nil { |
| + ir.local.Authorize = ir.authorizeChore |
| + } |
| + return ir |
| +} |
| + |
| +func (ir *installRun) cfg() moveCfg { return ir.w.cfg } |
| + |
| +// fault is the test seam after a side effect (the landing's points). |
| +func (ir *installRun) fault(point string) error { |
| + if f := ir.w.cfg.landFault; f != nil { |
| + return f(point) |
| + } |
| + return nil |
| +} |
| + |
| +// sync is each phase's first step: the kind's claim read, then its check at |
| +// phase (K5's revalidation). |
| +func (ir *installRun) sync(ctx context.Context, phase string) (hostclaim.Observed, error) { |
| + obs, err := ir.src.Sync(ctx) |
| + if err != nil { |
| + return obs, err |
| + } |
| + if err := ir.src.ClaimCheck(phase)(obs.Claim); err != nil { |
| + return obs, fmt.Errorf("move %s: %w", ir.g.ID, err) |
| + } |
| + return obs, nil |
| +} |
| + |
| +// seal reads the sealed set from the claim and requires every object it |
| +// names in this install's lane. |
| +func (ir *installRun) seal(c *hostclaim.Claim) (moves.Seal, error) { |
| + seal, err := ir.src.Seal(c) |
| + if err != nil { |
| + return seal, err |
| + } |
| + for _, class := range [][]moves.SealedObject{seal.Replica, seal.Files, seal.Timers} { |
| + for _, o := range class { |
| + if !strings.HasPrefix(o.Key, ir.src.Lane) { |
| + return seal, fmt.Errorf("%w: %s is not in the %s's lane %s", moves.ErrBadSeal, o.Key, ir.src.Name, ir.src.Lane) |
| + } |
| + } |
| + } |
| + return seal, nil |
| +} |
| + |
| +// transition writes one record transition. A conflict is someone else's |
| +// decision (a rollback, a stuck mark, a console write): logged, and the call |
| +// ends without error. |
| +func (ir *installRun) transition(ctx context.Context, what string, mut func(*provision.Record) error) (bool, error) { |
| + if _, err := provision.SetRecordMove(ctx, ir.s, ir.rec.Account, ir.rec.App, ir.rec.Host, mut); err != nil { |
| + if errors.Is(err, provision.ErrMoveConflict) { |
| + log.Printf("moves: %s: %s of %s refused (%v); the record moved on", ir.rec.Host, what, ir.g.ID, err) |
| + return false, nil |
| + } |
| + return false, fmt.Errorf("%s: %w", what, err) |
| + } |
| + return true, nil |
| +} |
| + |
| +// advance writes the record's step and then, for a landing, mirrors it into |
| +// the claim. A mirror that fails is logged: the next pass's sync writes it. |
| +func (ir *installRun) advance(ctx context.Context, from, to string, st installStamp) (bool, error) { |
| + name := ir.src.Name |
| + ok, err := ir.transition(ctx, "the "+name+" "+from+" to "+to, ir.src.Phase(from, to, st)) |
| + if !ok { |
| + return ok, err |
| + } |
| + log.Printf("moves: %s: %s %s is %s", ir.rec.Host, name, ir.g.ID, to) |
| + if err := ir.fault("after-record-" + to); err != nil { |
| + return true, err |
| + } |
| + if ir.src.Mirror != nil { |
| + if err := ir.src.Mirror(ctx, from, to); err != nil { |
| + log.Printf("moves: %s: the claim half of %s %s waits for the next pass: %v", ir.rec.Host, name, to, err) |
| + } |
| + } |
| + return true, nil |
| +} |
| + |
| +// refuse writes the kind's refusal; written is true when the record took it. |
| +func (ir *installRun) refuse(ctx context.Context, why installRefusal) (bool, error) { |
| + ok, sentence, err := ir.src.Refuse(ctx, why) |
| + if ok { |
| + ir.refused = sentence |
| + } |
| + return ok, err |
| +} |
| + |
| +func (ir *installRun) pastDeadline() bool { return ir.src.PastDeadline != nil && ir.src.PastDeadline() } |
| + |
| +// route is the host's registry row on this box. |
| +func (ir *installRun) route() (registry.Route, error) { |
| + rt, err := ir.cfg().Route(ir.rec.Host) |
| + if err != nil { |
| + return registry.Route{}, fmt.Errorf("read the registry row: %w", err) |
| + } |
| + if rt == nil { |
| + return registry.Route{}, fmt.Errorf("%s has no registry row on this box", ir.rec.Host) |
| + } |
| + return *rt, nil |
| +} |
| + |
| +func (ir *installRun) dbPath() string { return provision.DefaultDBPath(ir.cfg().DataDir, ir.rec.Host) } |
| + |
| +// journal is the install's own op journal, named by its generation. |
| +func (ir *installRun) journal() (*moves.Journal, error) { |
| + return moves.ReadJournal(ir.cfg().DataDir, ir.rec.Host, ir.src.GenerationOp) |
| +} |
| + |
| +func (ir *installRun) journalDir() string { |
| + return moves.JournalDir(ir.cfg().DataDir, ir.rec.Host, ir.src.GenerationOp) |
| +} |
| + |
| +// buildDir is BuildGeneration's scratch, <JournalDir>/build/ (ruling G34), |
| +// on the data disk so the result renames into DBPath. |
| +func (ir *installRun) buildDir() string { return filepath.Join(ir.journalDir(), "build") } |
| + |
| +// versioned is this box's own bucket by exact version. |
| +func (ir *installRun) versioned(ctx context.Context) (lease.VersionedStore, error) { |
| + cfg := ir.cfg() |
| + if cfg.Versioned == nil { |
| + return nil, errors.New("no versioned store on this box") |
| + } |
| + return cfg.Versioned(ctx, cfg.OwnBucket, cfg.OwnBucketRegion) |
| +} |
| + |
| +// parking (§7.6, D6): whatever an earlier stay of this incarnation left at |
| +// holder.location in this box's own bucket is parked before the build |
| +// writes beside it, one park-generation chore per generation (the origin as |
| +// @root), scoped to the location with the flip's time as the evidence |
| +// horizon. On the target that is an earlier stay's residue; on the source |
| +// of a dirty return it is the pre-move generations (§7.8, D6). Each chore is |
| +// run once per pass through the park engine, which takes the location's |
| +// lease itself (ErrPending is the expected answer: the acknowledgement waits |
| +// for the horizon, and is C4's runner's); the phase advances once every |
| +// chore's manifest exists and its deletion set is gone (Engine.Parked). |
| +// Per-generation chores are what let the acknowledgement finish after the |
| +// probe makes the new generation live here. |
| +// |
| +// A replay finds parking and continues from the claim: the residue list is |
| +// journalled so the chores are the same ones, the queue is idempotent per |
| +// generation, and Engine.Run resumes a chore from its manifest. |
| +func (ir *installRun) parking(ctx context.Context) (bool, error) { |
| + cfg := ir.cfg() |
| + host, id, gen := ir.rec.Host, ir.g.ID, ir.src.GenerationOp |
| + phase, next := ir.src.Phases[0], ir.src.Phases[1] |
| + obs, err := ir.sync(ctx, phase) |
| + if err != nil { |
| + return false, err |
| + } |
| + holder := *obs.Claim.Holder |
| + location := strings.TrimSuffix(holder.Location, "/") |
| + vs, err := ir.versioned(ctx) |
| + if err != nil { |
| + return false, err |
| + } |
| + j, err := ir.journal() |
| + if err != nil { |
| + return false, err |
| + } |
| + var gens []string |
| + if raw, ok := j.Steps[landStepResidue]; ok { |
| + if err := json.Unmarshal([]byte(raw), &gens); err != nil { |
| + return false, fmt.Errorf("the journal's residue list: %w", err) |
| + } |
| + } else { |
| + objs, err := vs.ListVersions(ctx, cfg.OwnBucket, location+"/") |
| + if err != nil { |
| + return false, fmt.Errorf("list %s: %w", location, err) |
| + } |
| + seen := map[string]bool{} |
| + for _, o := range objs { |
| + if o.Key == location+"/lock.json" { |
| + continue |
| + } |
| + g, ok := landingGenerationOf(location, o.Key) |
| + if !ok { |
| + return false, fmt.Errorf("%s is not an object of a generation this %s can classify", o.Key, ir.src.Name) |
| + } |
| + if g == gen || g == landingMarkerGeneration { |
| + // The install's own generation is never residue, and the |
| + // marker is superseded by the probe, never parked. |
| + continue |
| + } |
| + if g == "" { |
| + g = hostclaim.RootGeneration |
| + } |
| + seen[g] = true |
| + } |
| + for g := range seen { |
| + gens = append(gens, g) |
| + } |
| + sort.Strings(gens) |
| + body, err := json.Marshal(gens) |
| + if err != nil { |
| + return false, err |
| + } |
| + j.Phase = phase |
| + j.Mark(landStepResidue, string(body)) |
| + if err := j.Write(cfg.DataDir); err != nil { |
| + return false, err |
| + } |
| + } |
| + if len(gens) == 0 { |
| + return ir.advance(ctx, phase, next, installStamp{}) |
| + } |
| + if ir.rec.Move.FlippedAt == nil { |
| + return false, fmt.Errorf("move %s: the record has no flip time", id) |
| + } |
| + scope := hostclaim.ParkScope{Location: location, Incarnation: holder.Incarnation, EvidenceAt: ir.rec.Move.FlippedAt.UTC(), RequestLimit: landingRequestLimit} |
| + if obs, err = hostclaim.Update(ctx, ir.s, obs, hostclaim.QueueResidueParks(id, cfg.Region, scope, gens, obligationID)); err != nil { |
| + return false, fmt.Errorf("queue the residue parks: %w", err) |
| + } |
| + if err := ir.fault("after-residue-queued"); err != nil { |
| + return false, err |
| + } |
| + eng := ir.engine(vs) |
| + pending := 0 |
| + for _, o := range obs.Claim.Obligations { |
| + if o.Op != id || o.Kind != "park-generation" || o.Region != cfg.Region { |
| + continue |
| + } |
| + w := park.Work{Host: host, ID: o.ID, Op: id} |
| + if err := eng.Run(ctx, w); err != nil && !errors.Is(err, park.ErrPending) { |
| + return false, fmt.Errorf("park generation %s: %w", o.Generation, err) |
| + } |
| + parked, err := eng.Parked(ctx, w) |
| + if errors.Is(err, park.ErrChanged) { |
| + continue // acknowledged already |
| + } |
| + if err != nil { |
| + return false, fmt.Errorf("park generation %s: %w", o.Generation, err) |
| + } |
| + if !parked { |
| + pending++ |
| + } |
| + } |
| + if pending > 0 { |
| + ir.w.notice(host+"#parking", "moves: %s: %d residue park(s) are not complete yet", host, pending) |
| + return false, nil |
| + } |
| + return ir.advance(ctx, phase, next, installStamp{}) |
| +} |
| + |
| +// building (§7.6, §5.1 step 2): the generation built from the seal's pinned |
| +// replica versions under the location's lease (moves.BuildGeneration: the |
| +// intent, the snapshot, the anchor, the completed install manifest, every |
| +// one create-only), with the result left at <JournalDir>/build/<host>.db |
| +// for the restore. The manifest goes into the journal. |
| +// |
| +// A replay finds building and builds again: BuildGeneration reuses what it |
| +// finds under the generation's keys and refuses anything it did not make. |
| +func (ir *installRun) building(ctx context.Context) (bool, error) { |
| + cfg := ir.cfg() |
| + host, id, gen := ir.rec.Host, ir.g.ID, ir.src.GenerationOp |
| + phase, next := ir.src.Phases[1], ir.src.Phases[2] |
| + obs, err := ir.sync(ctx, phase) |
| + if err != nil { |
| + return false, err |
| + } |
| + holder := *obs.Claim.Holder |
| + held, waiting, err := ir.lease(ctx, holder) |
| + if waiting || err != nil { |
| + return false, err |
| + } |
| + seal, err := ir.seal(obs.Claim) |
| + if err != nil { |
| + return false, err |
| + } |
| + src := ir.src.Baseline |
| + if src == nil || seal.N != src.FinalTXID { |
| + return false, fmt.Errorf("move %s: the seal is at TXID %d and the record at %d", id, seal.N, srcTXID(src)) |
| + } |
| + vs, err := ir.versioned(ctx) |
| + if err != nil { |
| + return false, err |
| + } |
| + if err := held.Check(ctx); err != nil { |
| + return false, fmt.Errorf("the location's lease: %w", err) |
| + } |
| + // Every generation request runs under live lease authority (fix round |
| + // 1, item 3): the store checks the lease before each call, as the cut's |
| + // guarded writes check theirs, the requests run under a context the |
| + // lease's loss cancels (leaseContext, fix round 2), and the lease is |
| + // checked again before the build's completion is recorded. |
| + m, err := moves.BuildGeneration(leaseContext(ctx, held), moves.BuildInputs{Store: leasedStore{VersionedStore: vs, check: held.Check}, Bucket: cfg.OwnBucket, Source: seal.Replica, N: seal.N, |
| + GenerationPrefix: moves.GenerationPrefix(holder.Location, gen), Op: gen, Host: host, WorkDir: ir.buildDir(), Now: cfg.Now}) |
| + if err != nil { |
| + return false, fmt.Errorf("build the generation: %w", err) |
| + } |
| + if err := held.Check(ctx); err != nil { |
| + return false, fmt.Errorf("the location's lease after the build: %w", err) |
| + } |
| + body, err := json.Marshal(m) |
| + if err != nil { |
| + return false, err |
| + } |
| + j, err := ir.journal() |
| + if err != nil { |
| + return false, err |
| + } |
| + j.Phase = phase |
| + j.Mark(landStepInstall, string(body)) |
| + if err := j.Write(cfg.DataDir); err != nil { |
| + return false, err |
| + } |
| + if err := ir.fault("after-build"); err != nil { |
| + return false, err |
| + } |
| + log.Printf("moves: %s: generation %s built at TXID %d", host, gen, m.N) |
| + return ir.advance(ctx, phase, next, installStamp{}) |
| +} |
| + |
| +// published (§7.6): the build is complete and recorded; generation.json is |
| +// not written here (R10-3: the probe's cold admission publishes it). The |
| +// journal records the phase, the record advances to local and the location's |
| +// lease is released: the build was the only writer that needed it. |
| +func (ir *installRun) published(ctx context.Context) (bool, error) { |
| + phase, next := ir.src.Phases[2], ir.src.Phases[3] |
| + if _, err := ir.sync(ctx, phase); err != nil { |
| + return false, err |
| + } |
| + j, err := ir.journal() |
| + if err != nil { |
| + return false, err |
| + } |
| + if !j.Done(landStepInstall) { |
| + return false, fmt.Errorf("move %s: published without an install manifest in the journal", ir.g.ID) |
| + } |
| + j.Phase = phase |
| + j.Mark(landStepPublished, ir.cfg().Now().UTC().Format(time.RFC3339Nano)) |
| + if err := j.Write(ir.cfg().DataDir); err != nil { |
| + return false, err |
| + } |
| + ok, err := ir.advance(ctx, phase, next, installStamp{}) |
| + if ok { |
| + ir.releaseLease() |
| + } |
| + return ok, err |
| +} |
| + |
| +// local (§7.6), each sub-step journalled and skipped on a replay once done, |
| +// in this order (round-2 finding 8): (1) the installation record must be at |
| +// the holder's stamp; (2) what is there before is parked: on the target an |
| +// earlier stay's artifacts tagged S' through park.Local (which refuses a |
| +// path tagged with another stamp), or a park-local left with nothing to |
| +// retire is completed; on the source of a dirty return its own pre-move |
| +// installation (retirePreMove); (3) the sealed archive, read by exact |
| +// version and verified, is extracted into this install's own staging, |
| +// <JournalDir>/extract/, and then, inside the host guard, moved to the files |
| +// directory and <host>.env when nothing this install did not write is there |
| +// (installFiles: the install never deletes at the live paths, final review |
| +// finding 1; a retry clears only its staging); (4) the new paths are |
| +// registered under the holder's stamp; (5) the timers table is the sealed |
| +// snapshot with its registrations, the carried columns are set and the |
| +// restore-verification entry removed; (6) the prepared files manifest, |
| +// skipped with a log line until C8 exists; (7) the files directory is synced |
| +// and .files-gen written last, so its presence vouches for every step |
| +// before it. |
| +// |
| +// A replay finds local and finishes what the journal does not record; |
| +// nothing here is read by the probe before .files-gen exists. |
| +func (ir *installRun) local_(ctx context.Context) (bool, error) { |
| + cfg := ir.cfg() |
| + host, id, gen := ir.rec.Host, ir.g.ID, ir.src.GenerationOp |
| + phase, next := ir.src.Phases[3], ir.src.Phases[4] |
| + obs, err := ir.sync(ctx, phase) |
| + if err != nil { |
| + return false, err |
| + } |
| + holder := *obs.Claim.Holder |
| + src := ir.src.Baseline |
| + if src == nil || src.Files == nil { |
| + return false, fmt.Errorf("move %s: the record has no source baselines", id) |
| + } |
| + seal, err := ir.seal(obs.Claim) |
| + if err != nil { |
| + return false, err |
| + } |
| + j, err := ir.journal() |
| + if err != nil { |
| + return false, err |
| + } |
| + j.Phase = phase |
| + mark := func(step, value string) error { |
| + j.Mark(step, value) |
| + if err := j.Write(cfg.DataDir); err != nil { |
| + return err |
| + } |
| + return ir.fault("after-" + step) |
| + } |
| + captured, err := ir.capture(holder) |
| + if err != nil { |
| + return false, err |
| + } |
| + if !j.Done(landStepRetired) { |
| + if ir.src.Role == roleSource { |
| + if err := ir.retirePreMove(ctx, obs, captured, j, mark); err != nil { |
| + return false, err |
| + } |
| + } else if raw, ok := j.Steps[landStepPrevious]; ok { |
| + earlier, err := strconv.ParseUint(raw, 10, 64) |
| + if err != nil || earlier == 0 { |
| + return false, fmt.Errorf("the journal's previous stamp %q", raw) |
| + } |
| + var chore *hostclaim.Obligation |
| + for i := range obs.Claim.Obligations { |
| + if o := &obs.Claim.Obligations[i]; o.Op == id && o.Kind == "park-local" && o.Box == cfg.Box { |
| + chore = o |
| + } |
| + } |
| + switch { |
| + case chore != nil && hostclaim.InventoryHasStamp(obs.Claim, cfg.Box, earlier): |
| + if err := ir.local.RetireArtifacts(ctx, host, captured, chore.ID, id, earlier); err != nil { |
| + return false, fmt.Errorf("retire the earlier stay's artifacts: %w", err) |
| + } |
| + log.Printf("moves: %s: the earlier stay's artifacts (stamp %d) are parked", host, earlier) |
| + case chore != nil: |
| + if _, err := hostclaim.Update(ctx, ir.s, obs, hostclaim.CompleteLocalPark(id, cfg.Box, earlier)); err != nil { |
| + return false, fmt.Errorf("complete the local park: %w", err) |
| + } |
| + } |
| + } |
| + if err := mark(landStepRetired, ""); err != nil { |
| + return false, err |
| + } |
| + } |
| + dir, envPath := cfg.FilesDir(host) |
| + if !j.Done(landStepExtracted) { |
| + outcome, err := ir.installFiles(ctx, j, mark, captured, seal, *src.Files, dir, envPath) |
| + switch { |
| + case err != nil: |
| + return false, err |
| + case outcome == restoreMarked: |
| + return true, nil // the record changed: the refusal |
| + case outcome == restoreWaiting: |
| + return false, nil |
| + } |
| + } |
| + if !j.Done(landStepInventory) { |
| + paths := []string{host + ".db", host + ".db.gen", host, host + ".env"} |
| + if err := ir.local.RecordInventory(ctx, host, captured, "flat", paths, ir.src.ClaimCheck(phase)); err != nil { |
| + return false, fmt.Errorf("record the inventory: %w", err) |
| + } |
| + if err := mark(landStepInventory, ""); err != nil { |
| + return false, err |
| + } |
| + } |
| + if !j.Done(landStepTimers) { |
| + rows, err := ir.sealedTimers(ctx, seal, src.TimersSHA256) |
| + if err != nil { |
| + return false, err |
| + } |
| + if err := cfg.ReplaceTimers(ctx, host, rows); err != nil { |
| + return false, fmt.Errorf("replace the timers: %w", err) |
| + } |
| + if err := mark(landStepTimers, strconv.Itoa(len(rows))); err != nil { |
| + return false, err |
| + } |
| + } |
| + if !j.Done(landStepColumns) { |
| + if err := cfg.SetReplication(host, src.Replication); err != nil { |
| + return false, fmt.Errorf("set replication: %w", err) |
| + } |
| + if err := cfg.SetNextDue(host, src.NextDue); err != nil { |
| + return false, fmt.Errorf("set next_due: %w", err) |
| + } |
| + if err := mark(landStepColumns, ""); err != nil { |
| + return false, err |
| + } |
| + } |
| + if !j.Done(landStepVerify) { |
| + if err := cfg.RemoveVerifyEntry(ctx, ir.s, host); err != nil { |
| + return false, fmt.Errorf("remove the restore-verification entry: %w", err) |
| + } |
| + if err := mark(landStepVerify, ""); err != nil { |
| + return false, err |
| + } |
| + } |
| + if !j.Done(landStepFilesManifest) { |
| + // Spec rev 7.22 §7.6 (L17, R19-1): the prepared files manifest under |
| + // files/g-<generation>/ in the location, bound to {location, |
| + // g-<generation>, op, stamp}. Its build (C8, files.BuildPrepared) is |
| + // not on main; until it is, the step is recorded as skipped and the |
| + // probe's admission establishes the files baseline (Plan 3). |
| + log.Printf("moves: %s: %s", host, landingFilesManifestSkipped) |
| + if err := mark(landStepFilesManifest, landingFilesManifestSkipped); err != nil { |
| + return false, err |
| + } |
| + } |
| + if err := ir.fault("before-files-gen"); err != nil { |
| + return false, err |
| + } |
| + marker, err := json.Marshal(filesGenMarker{Op: gen, Incarnation: holder.Incarnation}) |
| + if err != nil { |
| + return false, err |
| + } |
| + placed, err := journalledStaged(j, landStepFilesPlaced) |
| + if err != nil { |
| + return false, err |
| + } |
| + // The marker goes into the files directory this install placed, and |
| + // only while the installation is still this install's (final review, |
| + // finding 2): inside the guard, the directory is the receipt's. |
| + if err := ir.guarded(ctx, captured, phase, func() error { |
| + if ok, err := sameIdentity(dir, placed.Files); err != nil || !ok { |
| + return fmt.Errorf("%w: %s is not the files directory this %s placed (%v)", errLandingChanged, dir, ir.src.Name, err) |
| + } |
| + if err := syncPath(dir); err != nil { |
| + return err |
| + } |
| + return writeFileAtomic(filepath.Join(dir, archive.MarkerName), marker, 0o644) |
| + }); err != nil { |
| + return false, fmt.Errorf("write %s: %w", archive.MarkerName, err) |
| + } |
| + if err := mark(landStepFilesGen, ""); err != nil { |
| + return false, err |
| + } |
| + log.Printf("moves: %s: files, environment, timers and columns installed; %s written", host, archive.MarkerName) |
| + return ir.advance(ctx, phase, next, installStamp{}) |
| +} |
| + |
| +// installArtifactPaths are an installation's artifacts at the flat paths |
| +// (G11), relative to the data directory: the database with its marker, |
| +// journal, quarantine and litestream shadow, the files directory and |
| +// <host>.env. A dirty return parks them before it installs; the source's |
| +// finalization parks them and the rendered config beside them. |
| +func installArtifactPaths(host string) []string { |
| + db := host + ".db" |
| + return []string{db, db + ".gen", db + "-wal", db + "-shm", db + ".corrupt", ".activator-" + db + "-litestream", host, host + ".env"} |
| +} |
| + |
| +// retirePreMove is a dirty return's local sub-step (2) (ruling J13): the |
| +// source's own pre-move installation leaves the flat paths before anything |
| +// of the return's is placed there, so placeStaged and restore find them |
| +// empty and never install over them. When the claim's inventory lists paths |
| +// of S, the source's installation stamp (the holder's after ReturnToSource), |
| +// they are retired through park.Local under the host guard against the |
| +// park-local the source's claimed queued (RetireArtifacts completes it); |
| +// with paths of S listed and no park-local owed it stops, since only the |
| +// claimed CAS queues it. A park-local owed with nothing of S listed (a replay |
| +// after the retirement) is completed. Then every pre-move artifact still at |
| +// the flat paths, registered or not, is parked by the moves-owned journalled |
| +// rename (H5(b), parkLocalArtifacts) into the engine's layout. Retired |
| +// stamps other than S are not the source's to find here: RecordInventory |
| +// refuses a path still listed under one, and the install stops there. |
| +func (ir *installRun) retirePreMove(ctx context.Context, obs hostclaim.Observed, captured hostguard.Captured, j *moves.Journal, mark func(step, value string) error) error { |
| + cfg := ir.cfg() |
| + host, id := ir.rec.Host, ir.g.ID |
| + stamp := captured.Stamp |
| + var chore *hostclaim.Obligation |
| + for i := range obs.Claim.Obligations { |
| + if o := &obs.Claim.Obligations[i]; o.Op == id && o.Kind == "park-local" && o.Box == cfg.Box { |
| + chore = o |
| + } |
| + } |
| + registered := hostclaim.InventoryHasStamp(obs.Claim, cfg.Box, stamp) |
| + switch { |
| + case registered && chore != nil: |
| + if err := ir.local.RetireArtifacts(ctx, host, captured, chore.ID, id, stamp); err != nil { |
| + return fmt.Errorf("retire the source's pre-move installation: %w", err) |
| + } |
| + log.Printf("moves: %s: the source's pre-move installation (stamp %d) is retired", host, stamp) |
| + case registered: |
| + return fmt.Errorf("move %s: this box's inventory lists the source's pre-move paths at stamp %d, and the claim owes no park-local for them", id, stamp) |
| + case chore != nil: |
| + if _, err := hostclaim.Update(ctx, ir.s, obs, hostclaim.CompleteLocalPark(id, cfg.Box, stamp)); err != nil { |
| + return fmt.Errorf("complete the source's local park: %w", err) |
| + } |
| + } |
| + if _, err := parkLocalArtifacts(ctx, j, mark, localArtifactPark{DataDir: cfg.DataDir, Incarnation: captured.Incarnation, Host: host, |
| + Op: ir.src.GenerationOp, Step: installStepPreMove, Paths: installArtifactPaths(host), NewID: obligationID, Fault: ir.fault}); err != nil { |
| + return fmt.Errorf("park the source's pre-move artifacts: %w", err) |
| + } |
| + return nil |
| +} |
| + |
| +// configured (§7.6, §9.1, §9.2): the held row rendered (RenderHeld, the |
| +// move-bound ack). A landing requires the render's bundle and overlay etags |
| +// to be the ones pinned at readiness (else the refusal, "configuration |
| +// changed during the move") and the pinned release live and proven for the |
| +// held row through Plan 1's T7 hook (ruling G4: "waiting for the root stage |
| +// pass" waits, anything else waits until the landing's deadline refuses). A |
| +// return renders the current inputs instead, never compared with |
| +// move.inputs, gates on the release live on this box, and records what it |
| +// rendered as Rollback.RecoveryInputs in the CAS that ends the phase |
| +// (J2-R2-5). Then the install the brief lists under `restored`: the built |
| +// database's sha256 against the manifest, lease.CheckDB, the rename into |
| +// DBPath (G34), <dbPath>.gen through hostbox's marker writer naming the |
| +// generation (ruling G2), and the CAS to restored. |
| +// |
| +// A replay finds configured and does it all again: the render and the |
| +// release check are idempotent reads and writes of the same bytes; a |
| +// database already at DBPath with the manifest's sha256 is this install's |
| +// own, so only the marker is written again. |
| +func (ir *installRun) configured(ctx context.Context) (bool, error) { |
| + cfg := ir.cfg() |
| + host, id := ir.rec.Host, ir.g.ID |
| + phase, next := ir.src.Phases[4], ir.src.Phases[5] |
| + obs, err := ir.sync(ctx, phase) |
| + if err != nil { |
| + return false, err |
| + } |
| + var pin, recovery *provision.MoveInputs |
| + if ir.src.Role == roleTarget { |
| + if pin = ir.src.PinInputs; pin == nil { |
| + return false, fmt.Errorf("move %s: the record has no pinned inputs", id) |
| + } |
| + } |
| + rt, err := ir.route() |
| + if err != nil { |
| + return false, err |
| + } |
| + hr, err := cfg.RenderHeld(ctx, ir.s, rt, ir.src.HoldOp) |
| + if err != nil { |
| + return false, fmt.Errorf("render the held row: %w", err) |
| + } |
| + want := "" |
| + if pin != nil { |
| + if !sameBundle(hr.BundleETags, pin.BundleETags) || hr.OverlayETag != pin.OverlayETag { |
| + log.Printf("moves: %s: the render's etags (%v, overlay %q) are not the pinned ones (%v, overlay %q)", host, hr.BundleETags, hr.OverlayETag, pin.BundleETags, pin.OverlayETag) |
| + return ir.refuse(ctx, refuseConfigChanged) |
| + } |
| + want = pin.Release |
| + } else { |
| + live, err := release.LiveVersion(cfg.OptDir, host) |
| + if err != nil { |
| + return false, fmt.Errorf("read the live release: %w", err) |
| + } |
| + bundle := make(map[string]string, len(hr.BundleETags)) |
| + for k, v := range hr.BundleETags { |
| + bundle[k] = v |
| + } |
| + recovery = &provision.MoveInputs{BundleETags: bundle, OverlayETag: hr.OverlayETag, Release: live} |
| + want = live |
| + } |
| + if c := cfg.ReleaseReady(ctx, ir.s, ir.rec, want); !c.OK { |
| + ir.w.notice(host+"#release", "moves: %s: release %s is not live and proven for the held row yet (%s); retrying", host, want, c.Reason) |
| + return false, nil |
| + } |
| + if err := ir.fault("after-release-ready"); err != nil { |
| + return false, err |
| + } |
| + // The render, the release gate and the install can outlast a landing's |
| + // deadline (fix round 1, item 4): it is checked again before the install |
| + // and right before the transition that names it complete. |
| + if ir.pastDeadline() { |
| + return ir.refuse(ctx, refuseDeadline) |
| + } |
| + sha, outcome, err := ir.restore(ctx, obs) |
| + if err != nil { |
| + return false, err |
| + } |
| + switch outcome { |
| + case restoreWaiting: |
| + return false, nil |
| + case restoreMarked: |
| + return true, nil // the record changed: the refusal, not an install |
| + } |
| + if ir.pastDeadline() { |
| + return ir.refuse(ctx, refuseDeadline) |
| + } |
| + return ir.advance(ctx, phase, next, installStamp{InstalledDBSHA256: sha, RecoveryInputs: recovery}) |
| +} |
| + |
| +// recoveryInstallSource binds the installer to a dirty return on the source |
| +// (§7.8 source table, the dirty branch; rulings J13, J14, J20): the record |
| +// rolling_back and dirty with the source at an install phase; the claim |
| +// restoring/claimed for this rollback with this box, the move's source, |
| +// executing at the flip's journalled source stamp S; the generation and the |
| +// hold named by the rollback id; the return lane moves/<host>/<rb id>/ in |
| +// this box's own bucket; the refusal MoveMarkStuck; the phase advance |
| +// MoveRollbackSourcePhase, stamping RecoveryInputs at the end of configured. |
| +// seal reads the return seal from the claim the phase works from, and |
| +// baseline is what it restores to (the return's evidence): both are the |
| +// caller's (Plan 4 Tasks 4 and 5). Production does not call it yet. |
| +func recoveryInstallSource(w *movesWorker, s release.Store, rec provision.Record, seal func(*hostclaim.Claim) (moves.Seal, error), baseline *provision.MoveSource) (installSource, error) { |
| + cfg := w.cfg |
| + m := rec.Move |
| + if m == nil || m.Legacy() || m.State != provision.MoveRollingBack || m.Rollback == nil || !m.Rollback.Dirty || !moves.ValidOpID(m.Rollback.ID) || |
| + !strings.HasPrefix(m.Rollback.ID, "rb_") { |
| + return installSource{}, errors.New("moves: only a dirty rollback's source installs a return") |
| + } |
| + if cfg.Box == "" || seal == nil || baseline == nil { |
| + return installSource{}, errors.New("moves: a return's install names this box, its seal and its baseline") |
| + } |
| + g := provision.MoveGuard{ID: m.ID, Incarnation: m.Incarnation} |
| + host, rb, box := rec.Host, m.Rollback.ID, cfg.Box |
| + claimCheck := func(string) func(*hostclaim.Claim) error { |
| + return func(c *hostclaim.Claim) error { |
| + if err := hostclaim.RequireMoveOp(g.ID, hostclaim.KindRestoring, hostclaim.PhaseClaimed)(c); err != nil { |
| + return err |
| + } |
| + if c.Op.Exec != box || len(c.Op.Participants) == 0 || c.Op.Participants[0] != box || c.Op.Intent["source"] != box { |
| + return fmt.Errorf("%w: the executor is %q, not this box as the move's source", hostclaim.ErrWrongState, c.Op.Exec) |
| + } |
| + if c.Op.Intent["rollback"] != rb { |
| + return fmt.Errorf("%w: the op is rollback %q, not %s", hostclaim.ErrWrongState, c.Op.Intent["rollback"], rb) |
| + } |
| + stamp, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64) |
| + if err != nil || stamp == 0 || c.Holder == nil || c.Holder.Stamp != stamp { |
| + return fmt.Errorf("%w: the holder is not at the source's pre-move stamp", hostclaim.ErrWrongState) |
| + } |
| + return nil |
| + } |
| + } |
| + return installSource{ |
| + OpID: g.ID, Host: host, Role: roleSource, Name: "return", |
| + Lane: moves.StagingPrefix(host, rb), Seal: seal, Baseline: baseline, |
| + GenerationOp: rb, HoldOp: rb, |
| + Phases: [6]string{provision.RbSourceParking, provision.RbSourceBuilding, provision.RbSourcePublished, |
| + provision.RbSourceLocal, provision.RbSourceConfigured, provision.RbSourceRestored}, |
| + Phase: func(from, to string, st installStamp) func(*provision.Record) error { |
| + var stamp func(*provision.MoveRollback) |
| + if st.RecoveryInputs != nil { |
| + in := st.RecoveryInputs |
| + stamp = func(r *provision.MoveRollback) { r.RecoveryInputs = in } |
| + } |
| + return provision.MoveRollbackSourcePhase(g, from, to, stamp) |
| + }, |
| + Sync: func(ctx context.Context) (hostclaim.Observed, error) { |
| + obs, err := hostclaim.Read(ctx, s, host) |
| + if err != nil { |
| + return obs, fmt.Errorf("read the host claim: %w", err) |
| + } |
| + if obs.Absent { |
| + return obs, fmt.Errorf("move %s has no host claim", g.ID) |
| + } |
| + if err := hostclaim.RequireOp(g.ID)(obs.Claim); err != nil { |
| + return obs, fmt.Errorf("move %s: %w", g.ID, err) |
| + } |
| + if h := obs.Claim.Holder; h == nil || h.Incarnation != g.Incarnation { |
| + return obs, fmt.Errorf("move %s: the host claim names another incarnation", g.ID) |
| + } |
| + return obs, nil |
| + }, |
| + ClaimCheck: claimCheck, |
| + RecordPhase: func(r *provision.Record) (string, error) { |
| + m := r.Move |
| + if m == nil || m.ID != g.ID || m.Incarnation != g.Incarnation || m.State != provision.MoveRollingBack || m.Rollback == nil || m.Rollback.ID != rb || !m.Rollback.Dirty { |
| + return "", fmt.Errorf("%w: the record is no longer the dirty return of %s", errLandingChanged, rb) |
| + } |
| + return m.Rollback.SourcePhase, nil |
| + }, |
| + Refuse: func(ctx context.Context, why installRefusal) (bool, string, error) { |
| + var reason string |
| + switch why { |
| + case refuseFilesOccupied: |
| + reason = provision.StuckReturnFilesOccupied |
| + case refuseInstallMismatch: |
| + reason = provision.StuckReturnInstallMismatch |
| + default: |
| + return false, "", fmt.Errorf("move %s: a return's install has no refusal %d", g.ID, why) |
| + } |
| + if _, err := provision.SetRecordMove(ctx, s, rec.Account, rec.App, host, provision.MoveMarkStuck(g, reason)); err != nil { |
| + if errors.Is(err, provision.ErrMoveConflict) { |
| + log.Printf("moves: %s: the stuck mark of %s refused (%v); the record moved on", host, rb, err) |
| + return false, "", nil |
| + } |
| + return false, "", fmt.Errorf("mark the return stuck: %w", err) |
| + } |
| + log.Printf("moves: %s: the return %s stops for an operator: %s", host, rb, reason) |
| + nudgeSerial(ctx, s) |
| + return true, reason, nil |
| + }, |
| + }, nil |
| +} |
diff --git a/internal/host/movesinstall_test.go b/internal/host/movesinstall_test.go| new file mode 100644 |
| index 000000000..0c9c162ad |
| --- /dev/null |
| +++ b/internal/host/movesinstall_test.go |
| @@ -0,0 +1,667 @@ |
| +package host |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "io/fs" |
| + "os" |
| + "path/filepath" |
| + "sort" |
| + "strings" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/appconfig" |
| + "github.com/carlosframework/platform/internal/hostbox" |
| + "github.com/carlosframework/platform/internal/hostclaim" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| + "github.com/carlosframework/platform/internal/lease" |
| + "github.com/carlosframework/platform/internal/moves" |
| + "github.com/carlosframework/platform/internal/moves/archive" |
| + "github.com/carlosframework/platform/internal/park" |
| + "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// The shared installer on the source (Plan 4 Task 3, rulings J13, J14, J20, |
| +// J24): a dirty return installs the generation the target sent back, from |
| +// the return lane in the source's own bucket, through the same six phases |
| +// the landing runs. The fixture is the landing harness driven to the flip, |
| +// then taken by provision's and hostclaim's own transitions to the point |
| +// Task 5's source recovery hands the install over: the record rolling_back, |
| +// dirty, the source at parking; the claim restoring/claimed with the source |
| +// executing at its pre-move stamp S; the source's installation record |
| +// active at S; and the source's own pre-move files still at the flat paths. |
| +// The return lane holds the staged seal's objects "as given" (J22), copied |
| +// by version into moves/<host>/<rb id>/ of the source's bucket. |
| + |
| +const p1RB = "rb_abcdefghijklmnopqrstuvwxyz" |
| + |
| +type returnHarness struct { |
| + *landingHarness |
| + sdir string // the source's data directory |
| + rcfg moveCfg |
| + rw *movesWorker |
| + seal moves.Seal // the return seal |
| + baseline provision.MoveSource |
| + srcRoute registry.Route |
| + asked []string // the releases ReleaseReady was asked about |
| + render appconfig.HeldRender |
| + sealOf func(*hostclaim.Claim) (moves.Seal, error) |
| + installed bool // stop the drive at restored |
| +} |
| + |
| +// newReturnHarness builds the fixture. registered lists the source's four |
| +// flat paths in the claim's inventory at S and queues its park-local, as |
| +// Task 5's claimed does when the inventory lists them (J13). |
| +func newReturnHarness(t *testing.T, registered bool) *returnHarness { |
| + t.Helper() |
| + ctx := context.Background() |
| + h := &returnHarness{landingHarness: newLandingHarness(t)} |
| + h.flip(t) |
| + h.sdir = h.cfg.DataDir |
| + rec := h.read(t, p1Host) |
| + h.baseline = *rec.Move.Source |
| + g := provision.MoveGuard{ID: p1ID, Incarnation: h.inc} |
| + |
| + // The return lane, as given: every sealed object copied by version into |
| + // moves/<host>/<rb id>/ of the source's bucket. |
| + staged := h.landingHarness.seal(t) |
| + lane, from := moves.StagingPrefix(p1Host, p1RB), moves.StagingPrefix(p1Host, p1ID) |
| + copyClass := func(objs []moves.SealedObject) []moves.SealedObject { |
| + var out []moves.SealedObject |
| + for _, o := range objs { |
| + key := lane + strings.TrimPrefix(o.Key, from) |
| + ver, _, err := h.vs.CopyVersion(ctx, "data-uk", o.Key, o.VersionID, "data-ie", key, true) |
| + if err != nil { |
| + t.Fatalf("copy %s into the return lane: %v", o.Key, err) |
| + } |
| + out = append(out, moves.SealedObject{Key: key, VersionID: ver, Size: o.Size}) |
| + } |
| + return out |
| + } |
| + h.seal = staged |
| + h.seal.Replica, h.seal.Files, h.seal.Timers = copyClass(staged.Replica), copyClass(staged.Files), copyClass(staged.Timers) |
| + if err := h.seal.Validate(); err != nil { |
| + t.Fatal(err) |
| + } |
| + h.sealOf = func(*hostclaim.Claim) (moves.Seal, error) { return h.seal, nil } |
| + |
| + // The record: the rollback, the target released dirty and returned, the |
| + // source claimed and at parking. |
| + now := h.clock.Now() |
| + for _, step := range []func(*provision.Record) error{ |
| + provision.MoveRollBack(g, p1RB, provision.RollbackLandingDeadline, now), |
| + provision.MoveRollbackTargetPhase(g, provision.RbTargetReleasing, provision.RbTargetReleased, func(rb *provision.MoveRollback) { |
| + rb.Dirty, rb.Evidence = true, "moves/bab/app/"+p1Host+"/"+p1ID+"/stopped-target.json" |
| + }), |
| + provision.MoveRollbackTargetPhase(g, provision.RbTargetReleased, provision.RbTargetReturning, nil), |
| + provision.MoveRollbackTargetPhase(g, provision.RbTargetReturning, provision.RbTargetReturned, nil), |
| + provision.MoveRollbackSourcePhase(g, provision.RbSourceWaiting, provision.RbSourceClaimed, nil), |
| + provision.MoveRollbackSourcePhase(g, provision.RbSourceClaimed, provision.RbSourceRestoring, nil), |
| + provision.MoveRollbackSourcePhase(g, provision.RbSourceRestoring, provision.RbSourceParking, nil), |
| + } { |
| + if _, err := provision.SetRecordMove(ctx, h.s, "bab", "app", p1Host, step); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + // The claim: the rollback, the target's release, the source's return. |
| + for _, step := range []func(*hostclaim.Claim) error{ |
| + hostclaim.RollBack(p1ID, p1RB), |
| + hostclaim.TargetReleased(p1ID, "uk-box", "uk", "stopped-target", hostclaim.ParkScope{Location: cutRoot, Incarnation: h.inc, EvidenceAt: now, RequestLimit: landingRequestLimit}, obligationID), |
| + hostclaim.ReturnToSource(p1ID, "ie-box", false, hostclaim.ReturnFence{}), |
| + } { |
| + obs, err := hostclaim.Read(ctx, h.s, p1Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := hostclaim.Update(ctx, h.s, obs, step); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if c := h.claim(t); c.Op.Kind != hostclaim.KindRestoring || c.Op.Phase != hostclaim.PhaseClaimed || c.Op.Exec != "ie-box" || c.Holder.Stamp != 1 { |
| + t.Fatalf("fixture claim: %+v holder %+v", c.Op, c.Holder) |
| + } |
| + // The source's installation record, active at S (Task 5's claimed). |
| + for _, d := range []string{"hostguard", ".parked"} { |
| + if err := os.MkdirAll(filepath.Join(h.sdir, d), 0o770); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + guard := hostguard.Guard{DataDir: h.sdir, LockDir: t.TempDir(), Claims: hostguard.StoreReader{Store: h.s}} |
| + if err := guard.Activate(ctx, p1Host, "ie-box", nil); err != nil { |
| + t.Fatal(err) |
| + } |
| + // The source's pre-move installation beside the files and env the cut |
| + // left: the database, its marker naming the source's generation, a WAL |
| + // and the litestream shadow. |
| + db := provision.DefaultDBPath(h.sdir, p1Host) |
| + for path, body := range map[string]string{db + ".gen": "cur\n", db + "-wal": "a stale wal", filepath.Join(h.sdir, ".activator-"+p1Host+".db-litestream", "generation"): "shadow"} { |
| + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + if registered { |
| + rewriteP1Claim(t, h.s, func(c *hostclaim.Claim) { |
| + c.Inventory = map[string]hostclaim.InventoryEntry{"ie-box": {Layout: "flat", Paths: map[string]uint64{p1Host + ".db": 1, p1Host + ".db.gen": 1, p1Host: 1, p1Host + ".env": 1}}} |
| + }) |
| + obs, err := hostclaim.Read(ctx, h.s, p1Host) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := hostclaim.Update(ctx, h.s, obs, hostclaim.QueueLocalPark(p1ID, "ie-box", obligationID)); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + h.srcRoute = registry.Route{Host: p1Host, Kind: "instance", App: "app", Account: "bab", Hibernate: true, Status: "restoring", HoldOp: p1RB, |
| + Socket: provision.DefaultSocket(p1Host), DBPath: db, Provisioned: true} |
| + // The render answers the current bundle, which is not the pinned one: a |
| + // recovery records what it rendered and never compares it. |
| + h.render = appconfig.HeldRender{Path: filepath.Join(h.sdir, p1Host+".json"), SHA256: "y", BundleETags: map[string]string{"default": "e9", "environment": ""}, OverlayETag: "o9"} |
| + h.rcfg = h.sourceCfg(t) |
| + return h |
| +} |
| + |
| +// sourceCfg is the source box's configuration with the installer armed: |
| +// the landing's hooks over the source's data directory, bucket and region. |
| +func (h *returnHarness) sourceCfg(t *testing.T) moveCfg { |
| + t.Helper() |
| + c := h.landingCfg(t) |
| + c.Region, c.Box, c.OwnBucket, c.OwnBucketRegion = "ie", "ie-box", "data-ie", "eu-west-1" |
| + c.DataDir = h.sdir |
| + c.OptDir = h.optDir |
| + c.OwnsRoute = func(host string) bool { return host == p1Host } |
| + c.Route = func(host string) (*registry.Route, error) { |
| + if host != p1Host { |
| + return nil, nil |
| + } |
| + rt := h.srcRoute |
| + return &rt, nil |
| + } |
| + c.FilesDir = func(host string) (string, string) { |
| + return filepath.Join(h.sdir, host), filepath.Join(h.sdir, host+".env") |
| + } |
| + c.Guard = hostguard.Guard{DataDir: h.sdir, LockDir: t.TempDir()} |
| + c.Local = park.Local{DataDir: h.sdir, Box: "ie-box"} |
| + c.RenderHeld = func(_ context.Context, _ release.Store, rt registry.Route, op string) (appconfig.HeldRender, error) { |
| + h.renders++ |
| + if op != p1RB || rt.HoldOp != p1RB { |
| + t.Errorf("rendered for %q over a row held by %q, want the rollback %s", op, rt.HoldOp, p1RB) |
| + } |
| + return h.render, nil |
| + } |
| + c.ReleaseReady = func(_ context.Context, _ release.Store, _ provision.Record, v string) moves.Check { |
| + h.asked = append(h.asked, v) |
| + return h.release |
| + } |
| + c.RemoveVerifyEntry = func(context.Context, release.Store, string) error { return nil } |
| + c.WriteGen = hostbox.WriteGenerationMarker |
| + return c |
| +} |
| + |
| +// install drives the installer the way Task 5's pass will: one phase per |
| +// call, the record re-read between them, until it stops advancing. It |
| +// answers the last call's recorded refusal and error. |
| +func (h *returnHarness) install(t *testing.T) (string, error) { |
| + t.Helper() |
| + ctx := context.Background() |
| + if h.rw == nil { |
| + h.rw = &movesWorker{mirrors: map[string]lease.MirrorStore{}} |
| + } |
| + h.rw.cfg = h.rcfg |
| + defer releaseOwnLease(h.rw, p1Host) |
| + for i := 0; i < 8; i++ { |
| + rec := h.read(t, p1Host) |
| + src, err := recoveryInstallSource(h.rw, h.s, rec, h.sealOf, &h.baseline) |
| + if err != nil { |
| + return "", err |
| + } |
| + advanced, refused, err := installGeneration(ctx, h.s, h.rw, rec, src) |
| + if err != nil || refused != "" || !advanced { |
| + return refused, err |
| + } |
| + } |
| + return "", nil |
| +} |
| + |
| +func (h *returnHarness) rollback(t *testing.T) (*provision.MoveState, *provision.MoveRollback) { |
| + t.Helper() |
| + m := h.read(t, p1Host).Move |
| + if m == nil || m.Rollback == nil { |
| + t.Fatalf("record: %+v", m) |
| + } |
| + return m, m.Rollback |
| +} |
| + |
| +func (h *returnHarness) journal(t *testing.T) *moves.Journal { |
| + t.Helper() |
| + j, err := moves.ReadJournal(h.sdir, p1Host, p1RB) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return j |
| +} |
| + |
| +// parkedFiles maps every regular file under the source's .parked/ to its body. |
| +func (h *returnHarness) parkedFiles(t *testing.T) map[string]string { |
| + t.Helper() |
| + out := map[string]string{} |
| + root := filepath.Join(h.sdir, ".parked") |
| + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { |
| + if err != nil || d.IsDir() { |
| + return err |
| + } |
| + body, err := os.ReadFile(p) |
| + if err != nil { |
| + return err |
| + } |
| + rel, _ := filepath.Rel(root, p) |
| + out[rel] = string(body) |
| + return nil |
| + }) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return out |
| +} |
| + |
| +func parkedHas(files map[string]string, suffix, body string) bool { |
| + for p, b := range files { |
| + if strings.HasSuffix(p, suffix) && b == body { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| +// installedReturn asserts the end state of a dirty return's install through |
| +// restored, on the source. |
| +func (h *returnHarness) installedReturn(t *testing.T, pinned *provision.MoveInputs) moves.InstallManifest { |
| + t.Helper() |
| + m, rb := h.rollback(t) |
| + if m.State != provision.MoveRollingBack || rb.SourcePhase != provision.RbSourceRestored || m.StuckReason != "" || !rb.Dirty { |
| + t.Fatalf("record: %+v %+v", m, rb) |
| + } |
| + // configured rendered the current inputs and recorded them, never |
| + // comparing them with the pin, which is untouched. |
| + want := provision.MoveInputs{BundleETags: map[string]string{"default": "e9", "environment": ""}, OverlayETag: "o9", Release: "v7"} |
| + if in := rb.RecoveryInputs; in == nil || in.Release != want.Release || in.OverlayETag != want.OverlayETag || !sameBundle(in.BundleETags, want.BundleETags) { |
| + t.Fatalf("recovery_inputs: %+v, want %+v", rb.RecoveryInputs, want) |
| + } |
| + if got := m.Inputs; got == nil || pinned == nil || got.Release != pinned.Release || !sameBundle(got.BundleETags, pinned.BundleETags) || got.OverlayETag != pinned.OverlayETag { |
| + t.Fatalf("move.inputs changed: %+v, was %+v", got, pinned) |
| + } |
| + if len(h.asked) == 0 { |
| + t.Fatal("the release gate was never asked") |
| + } |
| + for _, v := range h.asked { |
| + if v != "v7" { |
| + t.Fatalf("release gate asked about %v, want the live v7", h.asked) |
| + } |
| + } |
| + // The claim does not follow the source's install phases: it stays at |
| + // restoring/claimed with the source executing at S. |
| + c := h.claim(t) |
| + if c.Op.Kind != hostclaim.KindRestoring || c.Op.Phase != hostclaim.PhaseClaimed || c.Op.Exec != "ie-box" || c.Holder.Stamp != 1 { |
| + t.Fatalf("claim: %+v holder %+v", c.Op, c.Holder) |
| + } |
| + // The return generation, g-<rb id>, in the source's own bucket: the |
| + // build's four objects. |
| + prefix := moves.GenerationPrefix(cutRoot, p1RB) |
| + var keys []string |
| + for k := range h.f.buckets["data-ie"] { |
| + if strings.HasPrefix(k, prefix) { |
| + keys = append(keys, strings.TrimPrefix(k, prefix)) |
| + } |
| + } |
| + if len(keys) != 4 { |
| + t.Fatalf("return generation objects: %v", keys) |
| + } |
| + var im moves.InstallManifest |
| + if err := json.Unmarshal([]byte(h.journal(t).Steps[landStepInstall]), &im); err != nil || im.Op != p1RB || im.N != h.seal.N { |
| + t.Fatalf("install manifest in the return's journal: %v %+v", err, im) |
| + } |
| + dbPath := provision.DefaultDBPath(h.sdir, p1Host) |
| + // The pre-move WAL and litestream shadow are gone from the live paths |
| + // (checked before anything here opens the database, which makes its own). |
| + for _, p := range []string{dbPath + "-wal", filepath.Join(h.sdir, ".activator-"+p1Host+".db-litestream")} { |
| + if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("%s is still at the live path: %v", p, err) |
| + } |
| + } |
| + if got, err := fileSHA256(dbPath); err != nil || got != "sha256:"+im.DBSHA256 { |
| + t.Fatalf("the installed database: %v %s want %s", err, got, im.DBSHA256) |
| + } |
| + if got := countRowsForTest(t, dbPath); got != h.rows { |
| + t.Fatalf("the installed database holds %d rows, want %d", got, h.rows) |
| + } |
| + if gen, err := hostbox.ReadGenerationMarker(dbPath + ".gen"); err != nil || gen != p1RB { |
| + t.Fatalf("%s.gen: %q %v, want the return generation %s", dbPath, gen, err, p1RB) |
| + } |
| + body, err := os.ReadFile(filepath.Join(h.sdir, p1Host, archive.MarkerName)) |
| + var fg filesGenMarker |
| + if err != nil || json.Unmarshal(body, &fg) != nil || fg.Op != p1RB || fg.Incarnation != h.inc { |
| + t.Fatalf(".files-gen: %s %v", body, err) |
| + } |
| + if body, err := os.ReadFile(filepath.Join(h.sdir, p1Host, "upload.txt")); err != nil || string(body) != "hello" { |
| + t.Fatalf("the extracted files: %q %v", body, err) |
| + } |
| + // The new installation is registered at S, the source's own stamp. |
| + inv := c.Inventory["ie-box"] |
| + if inv.Layout != "flat" || len(inv.Paths) != 4 { |
| + t.Fatalf("inventory: %+v", inv) |
| + } |
| + for _, p := range []string{p1Host + ".db", p1Host + ".db.gen", p1Host, p1Host + ".env"} { |
| + if inv.Paths[p] != 1 { |
| + t.Fatalf("inventory path %s at %d, want S 1", p, inv.Paths[p]) |
| + } |
| + } |
| + // The pre-move generations at the location are residue parked in the |
| + // source's region, under the move's op. |
| + var gens []string |
| + for _, o := range c.Obligations { |
| + if o.Op == p1ID && o.Kind == "park-generation" && o.Region == "ie" && o.Park != nil && o.Park.Location == cutRoot { |
| + gens = append(gens, o.Generation) |
| + } |
| + if o.Kind == "park-local" { |
| + t.Fatalf("a park-local is still owed: %+v", o) |
| + } |
| + } |
| + sort.Strings(gens) |
| + if strings.Join(gens, " ") != "cur old" { |
| + t.Fatalf("residue chores: %v (%+v)", gens, c.Obligations) |
| + } |
| + for k := range h.f.buckets["data-ie"] { |
| + if strings.HasPrefix(k, cutRoot+"/g-cur/") || strings.HasPrefix(k, cutRoot+"/g-old/") { |
| + t.Fatalf("%s is still at the source's location", k) |
| + } |
| + } |
| + // Timers and the columns come from the return's baseline. |
| + if len(h.replaced) != 1 || moves.TimersDigest(h.replaced[0]) != h.baseline.TimersSHA256 { |
| + t.Fatalf("ReplaceTimers: %+v", h.replaced) |
| + } |
| + if len(h.repl) != 1 || h.repl[0] != h.baseline.Replication || len(h.nextDue) != 1 || h.nextDue[0] != h.baseline.NextDue { |
| + t.Fatalf("columns: %v %v", h.repl, h.nextDue) |
| + } |
| + if held := h.leases.holding(); len(held) != 0 { |
| + t.Fatalf("the install still holds %v", held) |
| + } |
| + return im |
| +} |
| + |
| +// A registered source (J13): RetireArtifacts(S) parks the inventoried |
| +// pre-move paths, the moves-owned rename parks what the inventory does not |
| +// list (the WAL, the litestream shadow), and only then is the return's |
| +// installation extracted and registered at S. |
| +func TestDirtyReturnInstallsUnderTheSourceStamp(t *testing.T) { |
| + h := newReturnHarness(t, true) |
| + pinned := h.read(t, p1Host).Move.Inputs |
| + if refused, err := h.install(t); err != nil || refused != "" { |
| + t.Fatalf("install: %q %v", refused, err) |
| + } |
| + h.installedReturn(t, pinned) |
| + parked := h.parkedFiles(t) |
| + for suffix, body := range map[string]string{ |
| + "/artifacts/" + p1Host + ".db": "the stopped database", |
| + "/artifacts/" + p1Host + ".db.gen": "cur\n", |
| + "/artifacts/" + p1Host + "/upload.txt": "hello", |
| + "/artifacts/" + p1Host + ".db-wal": "a stale wal", |
| + "/artifacts/.activator-" + p1Host + ".db-litestream/generation": "shadow", |
| + } { |
| + if !parkedHas(parked, suffix, body) { |
| + t.Fatalf("%s was not parked (%v)", suffix, parked) |
| + } |
| + } |
| + // The registered half was park.Local's: its journal is in the |
| + // retirement. The rest was the moves-owned rename, recorded by the op |
| + // that ran it, the return. |
| + var receipts int |
| + for p, b := range parked { |
| + if strings.HasSuffix(p, "/"+localParkReceiptName) { |
| + receipts++ |
| + var r localParkReceipt |
| + if err := json.Unmarshal([]byte(b), &r); err != nil || r.Op != p1RB { |
| + t.Fatalf("receipt %s: %s %v", p, b, err) |
| + } |
| + } |
| + } |
| + if receipts != 1 { |
| + t.Fatalf("%d moves-owned receipts", receipts) |
| + } |
| + if !h.journal(t).Done(installStepPreMove) || !h.journal(t).Done(landStepRetired) { |
| + t.Fatalf("journal: %v", h.journal(t).Steps) |
| + } |
| + // restored is the probe's (Task 9): another call changes nothing. |
| + before := h.read(t, p1Host).Move.UpdatedAt |
| + if refused, err := h.install(t); err != nil || refused != "" { |
| + t.Fatalf("install at restored: %q %v", refused, err) |
| + } |
| + if after := h.read(t, p1Host).Move; !after.UpdatedAt.Equal(before) { |
| + t.Fatalf("a call at restored wrote the record: %+v", after) |
| + } |
| +} |
| + |
| +// An unregistered source (Paul's H5(b)): no park-local is owed, and the |
| +// moves-owned rename parks every pre-move artifact before extraction. |
| +func TestDirtyReturnParksAnUnregisteredSource(t *testing.T) { |
| + h := newReturnHarness(t, false) |
| + pinned := h.read(t, p1Host).Move.Inputs |
| + if refused, err := h.install(t); err != nil || refused != "" { |
| + t.Fatalf("install: %q %v", refused, err) |
| + } |
| + h.installedReturn(t, pinned) |
| + parked := h.parkedFiles(t) |
| + for suffix, body := range map[string]string{ |
| + "/artifacts/" + p1Host + ".db": "the stopped database", |
| + "/artifacts/" + p1Host + ".db.gen": "cur\n", |
| + "/artifacts/" + p1Host + ".env": "APP_MODE=prod\n", |
| + "/artifacts/" + p1Host + "/upload.txt": "hello", |
| + "/artifacts/" + p1Host + ".db-wal": "a stale wal", |
| + } { |
| + if !parkedHas(parked, suffix, body) { |
| + t.Fatalf("%s was not parked (%v)", suffix, parked) |
| + } |
| + } |
| +} |
| + |
| +// A crash at each journalled step of the pre-move park and of the install |
| +// is finished by the next call, from the journal, with nothing parked twice. |
| +func TestDirtyReturnReplaysFromTheJournal(t *testing.T) { |
| + for _, point := range []string{"after-retired", "after-" + installStepPreMove + "-plan", "after-" + installStepPreMove, "after-staged", "after-files-rename", "after-record-local", "after-db-rename", "after-record-restored"} { |
| + t.Run(point, func(t *testing.T) { |
| + h := newReturnHarness(t, false) |
| + pinned := h.read(t, p1Host).Move.Inputs |
| + h.landAt = point |
| + if _, err := h.install(t); !errors.Is(err, errCrash) { |
| + t.Fatalf("the crash at %s: %v (points %v)", point, err, h.landHits) |
| + } |
| + h.rw = nil |
| + h.leases.expire() |
| + if refused, err := h.install(t); err != nil || refused != "" { |
| + t.Fatalf("the replay: %q %v", refused, err) |
| + } |
| + h.installedReturn(t, pinned) |
| + if n := len(h.parkedFiles(t)); n == 0 { |
| + t.Fatal("nothing parked") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// The source's own pre-move files are installed over only once parked: a |
| +// files path that holds anything after the park is not the return's, and |
| +// the install stops for an operator (stuck, never a landing's rollback) |
| +// with nothing moved; a marker that names another generation is never |
| +// replaced. |
| +func TestDirtyReturnNeverInstallsOverWhatItDidNotPark(t *testing.T) { |
| + t.Run("files", func(t *testing.T) { |
| + h := newReturnHarness(t, false) |
| + sentinel := filepath.Join(h.sdir, p1Host, "sentinel.txt") |
| + h.landFault = func(p string) error { |
| + if p == "after-"+landStepRetired { |
| + if err := os.MkdirAll(filepath.Dir(sentinel), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + return os.WriteFile(sentinel, []byte("not the return's"), 0o644) |
| + } |
| + return nil |
| + } |
| + refused, err := h.install(t) |
| + if err != nil || refused != provision.StuckReturnFilesOccupied { |
| + t.Fatalf("install: %q %v", refused, err) |
| + } |
| + m, rb := h.rollback(t) |
| + if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnFilesOccupied || rb.StuckAt != provision.RbSourceLocal || m.Reason != "" { |
| + t.Fatalf("record: %+v %+v", m, rb) |
| + } |
| + if body, err := os.ReadFile(sentinel); err != nil || string(body) != "not the return's" { |
| + t.Fatalf("the sentinel: %q %v", body, err) |
| + } |
| + if _, err := os.Stat(filepath.Join(h.sdir, p1Host, "upload.txt")); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("the return's files were placed beside the sentinel: %v", err) |
| + } |
| + // A stuck install is not driven on. |
| + if _, err := h.install(t); err == nil { |
| + t.Fatal("the installer ran a stuck record") |
| + } |
| + }) |
| + t.Run("marker", func(t *testing.T) { |
| + h := newReturnHarness(t, true) |
| + gen := provision.DefaultDBPath(h.sdir, p1Host) + ".gen" |
| + h.landFault = func(p string) error { |
| + if p == "after-"+landStepRetired { |
| + return os.WriteFile(gen, []byte("cur\n"), 0o600) |
| + } |
| + return nil |
| + } |
| + if _, err := h.install(t); !errors.Is(err, errLandingChanged) { |
| + t.Fatalf("install over the pre-move marker: %v", err) |
| + } |
| + if m, rb := h.rollback(t); m.State != provision.MoveRollingBack || rb.SourcePhase != provision.RbSourceConfigured { |
| + t.Fatalf("record: %+v %+v", m, rb) |
| + } |
| + if body, err := os.ReadFile(gen); err != nil || string(body) != "cur\n" { |
| + t.Fatalf("the marker was replaced: %q %v", body, err) |
| + } |
| + if _, err := os.Stat(provision.DefaultDBPath(h.sdir, p1Host)); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("a database was installed beside the marker: %v", err) |
| + } |
| + }) |
| +} |
| + |
| +// A return build that does not verify stops the install at configured, |
| +// stuck, with nothing installed. |
| +func TestDirtyReturnCorruptBuildIsStuck(t *testing.T) { |
| + h := newReturnHarness(t, true) |
| + h.landFault = func(point string) error { |
| + if point == "before-db-verify" { |
| + f, err := os.OpenFile(filepath.Join(h.sdir, ".moves", p1Host, p1RB, "build", p1Host+".db"), os.O_APPEND|os.O_WRONLY, 0o600) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer f.Close() |
| + if _, err := f.Write([]byte("corrupt")); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + return nil |
| + } |
| + refused, err := h.install(t) |
| + if err != nil || refused != provision.StuckReturnInstallMismatch { |
| + t.Fatalf("install: %q %v", refused, err) |
| + } |
| + m, rb := h.rollback(t) |
| + if m.State != provision.MoveStuck || m.StuckReason != provision.StuckReturnInstallMismatch || rb.StuckAt != provision.RbSourceConfigured || rb.RecoveryInputs != nil { |
| + t.Fatalf("record: %+v %+v", m, rb) |
| + } |
| + dbPath := provision.DefaultDBPath(h.sdir, p1Host) |
| + for _, p := range []string{dbPath, dbPath + ".gen"} { |
| + if _, err := os.Stat(p); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("%s exists after a refused install: %v", p, err) |
| + } |
| + } |
| +} |
| + |
| +// The claim check and the record check are the recovery's own (J13): a |
| +// claim with another executor, or at another kind or phase, is refused |
| +// before anything is written, and a clean rollback has nothing to install. |
| +func TestDirtyReturnRequiresItsClaimAndRecord(t *testing.T) { |
| + for name, edit := range map[string]func(*hostclaim.Claim){ |
| + "another executor": func(c *hostclaim.Claim) { c.Op.Exec = "uk-box" }, |
| + "another phase": func(c *hostclaim.Claim) { c.Op.Phase = hostclaim.PhaseProbing }, |
| + "another rollback": func(c *hostclaim.Claim) { c.Op.Intent["rollback"] = "rb_bcdefghijklmnopqrstuvwxya" }, |
| + "another holder": func(c *hostclaim.Claim) { c.Holder.Stamp = 7 }, |
| + "a landing op kind": func(c *hostclaim.Claim) { c.Op.Kind = hostclaim.KindLanding }, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + h := newReturnHarness(t, true) |
| + rewriteP1Claim(t, h.s, edit) |
| + before := h.claim(t) |
| + if _, err := h.install(t); err == nil { |
| + t.Fatal("installed under the wrong claim") |
| + } |
| + if _, rb := h.rollback(t); rb.SourcePhase != provision.RbSourceParking { |
| + t.Fatalf("the record advanced: %+v", rb) |
| + } |
| + if after := h.claim(t); len(after.Obligations) != len(before.Obligations) { |
| + t.Fatalf("chores queued: %+v", after.Obligations) |
| + } |
| + }) |
| + } |
| + t.Run("a clean rollback", func(t *testing.T) { |
| + h := newReturnHarness(t, true) |
| + rec := h.read(t, p1Host) |
| + rec.Move.Rollback.Dirty = false |
| + if _, err := recoveryInstallSource(&movesWorker{cfg: h.rcfg}, h.s, rec, h.sealOf, &h.baseline); err == nil { |
| + t.Fatal("a clean rollback has an install source") |
| + } |
| + }) |
| + t.Run("a seal outside the lane", func(t *testing.T) { |
| + // A usable seal over the move's own staging lane in the source's |
| + // bucket: every object readable at its version, but not the return's. |
| + h := newReturnHarness(t, true) |
| + ctx := context.Background() |
| + other := h.landingHarness.seal(t) |
| + for _, class := range [][]moves.SealedObject{other.Replica, other.Files, other.Timers} { |
| + for i, o := range class { |
| + ver, _, err := h.vs.CopyVersion(ctx, "data-uk", o.Key, o.VersionID, "data-ie", o.Key, true) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + class[i].VersionID = ver |
| + } |
| + } |
| + h.sealOf = func(*hostclaim.Claim) (moves.Seal, error) { return other, nil } |
| + if _, err := h.install(t); !errors.Is(err, moves.ErrBadSeal) { |
| + t.Fatalf("built from the move's staging lane, not the return lane: %v", err) |
| + } |
| + if n := len(h.vs.callsOn(moves.GenerationPrefix(cutRoot, p1RB))); n != 0 { |
| + t.Fatalf("the build ran: %d calls", n) |
| + } |
| + }) |
| +} |
| + |
| +// J24: the stand-in view's authoritative region is the other side's. A |
| +// landing works before the target's probe is admitted, so the source's |
| +// region (From) is authoritative; a dirty return works before the |
| +// recovery's admission, so the target's (To) is. |
| +func TestStandInViewTakesTheRole(t *testing.T) { |
| + m := &provision.MoveState{From: "ie", To: "uk"} |
| + obs := hostclaim.Observed{ETag: "e", Claim: &hostclaim.Claim{Holder: &hostclaim.Holder{Location: cutRoot + "/"}}} |
| + for role, want := range map[string][2]string{roleTarget: {"uk", "ie"}, roleSource: {"ie", "uk"}} { |
| + v, err := standInView(want[0], role, m)(context.Background(), obs) |
| + if err != nil || v.Region != want[0] || v.AuthoritativeRegion != want[1] { |
| + t.Fatalf("%s: %+v %v", role, v, err) |
| + } |
| + if g, ok := v.GenerationOf(park.Ref{Key: cutRoot + "/g-" + p1RB + "/0000/x.ltx"}); !ok || g != p1RB { |
| + t.Fatalf("%s: generation %q %v", role, g, ok) |
| + } |
| + } |
| + if _, err := standInView("ie", "", m)(context.Background(), obs); !errors.Is(err, park.ErrUnsafe) { |
| + t.Fatalf("no role: %v", err) |
| + } |
| +} |
diff --git a/internal/host/movesland.go b/internal/host/movesland.go| index d9caab4b2..f7ed7e7ee 100644 |
| --- a/internal/host/movesland.go |
| +++ b/internal/host/movesland.go |
| @@ -8,18 +8,13 @@ import ( |
| "log" |
| "os" |
| "path/filepath" |
| - "sort" |
| "strconv" |
| - "strings" |
| "time" |
| "github.com/carlosframework/platform/internal/hostclaim" |
| "github.com/carlosframework/platform/internal/hostguard" |
| "github.com/carlosframework/platform/internal/hostwrite" |
| - "github.com/carlosframework/platform/internal/lease" |
| "github.com/carlosframework/platform/internal/moves" |
| - "github.com/carlosframework/platform/internal/moves/archive" |
| - "github.com/carlosframework/platform/internal/park" |
| "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| @@ -35,7 +30,10 @@ import ( |
| // decision, the claim follows it). Nothing is carried in memory between |
| // passes but the own-location lease (building through published): every |
| // phase resumes from the record, the claim, this box's own bucket and the |
| -// op journal (moves.Journal, <DataDir>/.moves/<host>/<op>/). |
| +// op journal (moves.Journal, <DataDir>/.moves/<host>/<op>/). checking is |
| +// the landing's own; parking to restored run in the generation installer |
| +// the dirty return shares (movesinstall.go, Plan 4 Task 3), bound to this |
| +// landing by installSource. |
| // |
| // Phase names (ruling G8, the cut's convention). A record at X means every |
| // phase before X is complete and X's own work may have begun; a replay of X |
| @@ -127,7 +125,6 @@ type landingRun struct { |
| rec provision.Record |
| g provision.MoveGuard |
| guard hostguard.Guard |
| - local park.Local |
| } |
| // landingArmed reports whether this box can run the landing: every hook |
| @@ -152,14 +149,6 @@ func (w *movesWorker) landingPass(ctx context.Context, s release.Store, rec prov |
| if l.guard.Claims == nil { |
| l.guard.Claims = hostguard.StoreReader{Store: s} |
| } |
| - l.local = cfg.Local |
| - l.local.Guard, l.local.Claims = l.guard, s |
| - if l.local.Box == "" { |
| - l.local.Box = cfg.Box |
| - } |
| - if l.local.Authorize == nil { |
| - l.local.Authorize = l.authorizeChore |
| - } |
| defer l.settleLease() |
| for i := 0; i < landingStepsPerPass; i++ { |
| advanced, err := l.step(ctx) |
| @@ -205,16 +194,10 @@ func (l *landingRun) step(ctx context.Context) (bool, error) { |
| switch m.LandingPhase { |
| case provision.LandChecking: |
| return l.checking(ctx) |
| - case provision.LandParking: |
| - return l.parking(ctx) |
| - case provision.LandBuilding: |
| - return l.building(ctx) |
| - case provision.LandPublished: |
| - return l.published(ctx) |
| - case provision.LandLocal: |
| - return l.local_(ctx) |
| - case provision.LandConfigured: |
| - return l.configured(ctx) |
| + case provision.LandParking, provision.LandBuilding, provision.LandPublished, provision.LandLocal, provision.LandConfigured: |
| + // parking to restored are the shared installer's (movesinstall.go). |
| + advanced, _, err := installGeneration(ctx, l.s, l.w, l.rec, l.installSource()) |
| + return advanced, err |
| case provision.LandRestored: |
| // The install is complete and the probe's (Plan 3). The one thing |
| // still this pass's is the claim half of the last CAS: a crash after |
| @@ -426,23 +409,6 @@ func (l *landingRun) journal() (*moves.Journal, error) { |
| return moves.ReadJournal(l.cfg().DataDir, l.rec.Host, l.g.ID) |
| } |
| -func (l *landingRun) journalDir() string { |
| - return moves.JournalDir(l.cfg().DataDir, l.rec.Host, l.g.ID) |
| -} |
| - |
| -// buildDir is BuildGeneration's scratch, <JournalDir>/build/ (ruling G34), |
| -// on the data disk so the result renames into DBPath. |
| -func (l *landingRun) buildDir() string { return filepath.Join(l.journalDir(), "build") } |
| - |
| -// versioned is this box's own bucket by exact version. |
| -func (l *landingRun) versioned(ctx context.Context) (lease.VersionedStore, error) { |
| - cfg := l.cfg() |
| - if cfg.Versioned == nil { |
| - return nil, errors.New("no versioned store on this box") |
| - } |
| - return cfg.Versioned(ctx, cfg.OwnBucket, cfg.OwnBucketRegion) |
| -} |
| - |
| // ensureLocalDirs creates the two directories the host guard and park.Local |
| // require under the data directory (hostguard/ for the installation record, |
| // .parked/ for retired artifacts). On a box with claims on they exist |
| @@ -594,195 +560,6 @@ func (l *landingRun) checking(ctx context.Context) (bool, error) { |
| return l.advance(ctx, provision.LandChecking, provision.LandParking, nil) |
| } |
| -// parking (§7.6, D6): whatever an earlier stay of this incarnation left at |
| -// holder.location in this box's own bucket is parked before the build |
| -// writes beside it, one park-generation chore per generation (the origin as |
| -// @root), scoped to the location with the flip's time as the evidence |
| -// horizon. Each chore is run once per pass through the park engine, which |
| -// takes the location's lease itself (ErrPending is the expected answer: the |
| -// acknowledgement waits for the horizon, and is C4's runner's); the phase |
| -// advances once every chore's manifest exists and its deletion set is gone |
| -// (Engine.Parked). Per-generation chores are what let the acknowledgement |
| -// finish after the probe makes the new generation live here. |
| -// |
| -// A replay finds parking and continues from the claim: the residue list is |
| -// journalled so the chores are the same ones, the queue is idempotent per |
| -// generation, and Engine.Run resumes a chore from its manifest. |
| -func (l *landingRun) parking(ctx context.Context) (bool, error) { |
| - cfg := l.cfg() |
| - host, id := l.rec.Host, l.g.ID |
| - obs, err := l.syncClaimPhase(ctx) |
| - if err != nil { |
| - return false, err |
| - } |
| - if err := l.requirePhase(obs, provision.LandParking); err != nil { |
| - return false, err |
| - } |
| - holder := *obs.Claim.Holder |
| - location := strings.TrimSuffix(holder.Location, "/") |
| - vs, err := l.versioned(ctx) |
| - if err != nil { |
| - return false, err |
| - } |
| - j, err := l.journal() |
| - if err != nil { |
| - return false, err |
| - } |
| - var gens []string |
| - if raw, ok := j.Steps[landStepResidue]; ok { |
| - if err := json.Unmarshal([]byte(raw), &gens); err != nil { |
| - return false, fmt.Errorf("the journal's residue list: %w", err) |
| - } |
| - } else { |
| - objs, err := vs.ListVersions(ctx, cfg.OwnBucket, location+"/") |
| - if err != nil { |
| - return false, fmt.Errorf("list %s: %w", location, err) |
| - } |
| - seen := map[string]bool{} |
| - for _, o := range objs { |
| - if o.Key == location+"/lock.json" { |
| - continue |
| - } |
| - g, ok := landingGenerationOf(location, o.Key) |
| - if !ok { |
| - return false, fmt.Errorf("%s is not an object of a generation this landing can classify", o.Key) |
| - } |
| - if g == id || g == landingMarkerGeneration { |
| - // The landing's own generation is never residue, and the |
| - // marker is superseded by the probe, never parked. |
| - continue |
| - } |
| - if g == "" { |
| - g = hostclaim.RootGeneration |
| - } |
| - seen[g] = true |
| - } |
| - for g := range seen { |
| - gens = append(gens, g) |
| - } |
| - sort.Strings(gens) |
| - body, err := json.Marshal(gens) |
| - if err != nil { |
| - return false, err |
| - } |
| - j.Phase = provision.LandParking |
| - j.Mark(landStepResidue, string(body)) |
| - if err := j.Write(cfg.DataDir); err != nil { |
| - return false, err |
| - } |
| - } |
| - if len(gens) == 0 { |
| - return l.advance(ctx, provision.LandParking, provision.LandBuilding, nil) |
| - } |
| - if l.rec.Move.FlippedAt == nil { |
| - return false, fmt.Errorf("move %s: the record has no flip time", id) |
| - } |
| - scope := hostclaim.ParkScope{Location: location, Incarnation: holder.Incarnation, EvidenceAt: l.rec.Move.FlippedAt.UTC(), RequestLimit: landingRequestLimit} |
| - if obs, err = hostclaim.Update(ctx, l.s, obs, hostclaim.QueueResidueParks(id, cfg.Region, scope, gens, obligationID)); err != nil { |
| - return false, fmt.Errorf("queue the residue parks: %w", err) |
| - } |
| - if err := l.fault("after-residue-queued"); err != nil { |
| - return false, err |
| - } |
| - eng := l.engine(vs) |
| - pending := 0 |
| - for _, o := range obs.Claim.Obligations { |
| - if o.Op != id || o.Kind != "park-generation" || o.Region != cfg.Region { |
| - continue |
| - } |
| - w := park.Work{Host: host, ID: o.ID, Op: id} |
| - if err := eng.Run(ctx, w); err != nil && !errors.Is(err, park.ErrPending) { |
| - return false, fmt.Errorf("park generation %s: %w", o.Generation, err) |
| - } |
| - parked, err := eng.Parked(ctx, w) |
| - if errors.Is(err, park.ErrChanged) { |
| - continue // acknowledged already |
| - } |
| - if err != nil { |
| - return false, fmt.Errorf("park generation %s: %w", o.Generation, err) |
| - } |
| - if !parked { |
| - pending++ |
| - } |
| - } |
| - if pending > 0 { |
| - l.w.notice(host+"#parking", "moves: %s: %d residue park(s) are not complete yet", host, pending) |
| - return false, nil |
| - } |
| - return l.advance(ctx, provision.LandParking, provision.LandBuilding, nil) |
| -} |
| - |
| -// building (§7.6, §5.1 step 2): the generation built from the seal's pinned |
| -// replica versions under the location's lease (moves.BuildGeneration: the |
| -// intent, the snapshot, the anchor, the completed install manifest, every |
| -// one create-only), with the result left at <JournalDir>/build/<host>.db |
| -// for the restore. The manifest goes into the journal. |
| -// |
| -// A replay finds building and builds again: BuildGeneration reuses what it |
| -// finds under the generation's keys and refuses anything it did not make. |
| -func (l *landingRun) building(ctx context.Context) (bool, error) { |
| - cfg := l.cfg() |
| - host, id := l.rec.Host, l.g.ID |
| - obs, err := l.syncClaimPhase(ctx) |
| - if err != nil { |
| - return false, err |
| - } |
| - if err := l.requirePhase(obs, provision.LandBuilding); err != nil { |
| - return false, err |
| - } |
| - holder := *obs.Claim.Holder |
| - held, waiting, err := l.lease(ctx, holder) |
| - if waiting || err != nil { |
| - return false, err |
| - } |
| - seal, err := journalledSeal(obs.Claim) |
| - if err != nil { |
| - return false, err |
| - } |
| - src := l.rec.Move.Source |
| - if src == nil || seal.N != src.FinalTXID { |
| - return false, fmt.Errorf("move %s: the seal is at TXID %d and the record at %d", id, seal.N, srcTXID(src)) |
| - } |
| - vs, err := l.versioned(ctx) |
| - if err != nil { |
| - return false, err |
| - } |
| - if err := held.Check(ctx); err != nil { |
| - return false, fmt.Errorf("the location's lease: %w", err) |
| - } |
| - // Every generation request runs under live lease authority (fix round |
| - // 1, item 3): the store checks the lease before each call, as the cut's |
| - // guarded writes check theirs, the requests run under a context the |
| - // lease's loss cancels (leaseContext, fix round 2), and the lease is |
| - // checked again before the build's completion is recorded. |
| - m, err := moves.BuildGeneration(leaseContext(ctx, held), moves.BuildInputs{Store: leasedStore{VersionedStore: vs, check: held.Check}, Bucket: cfg.OwnBucket, Source: seal.Replica, N: seal.N, |
| - GenerationPrefix: moves.GenerationPrefix(holder.Location, id), Op: id, Host: host, WorkDir: l.buildDir(), Now: cfg.Now}) |
| - if err != nil { |
| - return false, fmt.Errorf("build the generation: %w", err) |
| - } |
| - if err := held.Check(ctx); err != nil { |
| - return false, fmt.Errorf("the location's lease after the build: %w", err) |
| - } |
| - body, err := json.Marshal(m) |
| - if err != nil { |
| - return false, err |
| - } |
| - j, err := l.journal() |
| - if err != nil { |
| - return false, err |
| - } |
| - j.Phase = provision.LandBuilding |
| - j.Mark(landStepInstall, string(body)) |
| - if err := j.Write(cfg.DataDir); err != nil { |
| - return false, err |
| - } |
| - if err := l.fault("after-build"); err != nil { |
| - return false, err |
| - } |
| - log.Printf("moves: %s: generation %s built at TXID %d", host, id, m.N) |
| - return l.advance(ctx, provision.LandBuilding, provision.LandPublished, nil) |
| -} |
| - |
| func srcTXID(src *provision.MoveSource) uint64 { |
| if src == nil { |
| return 0 |
| @@ -790,287 +567,64 @@ func srcTXID(src *provision.MoveSource) uint64 { |
| return src.FinalTXID |
| } |
| -// published (§7.6): the build is complete and recorded; generation.json is |
| -// not written here (R10-3: the probe's cold admission publishes it). The |
| -// journal records the phase, the record advances to local and the location's |
| -// lease is released: the build was the only writer that needed it. |
| -func (l *landingRun) published(ctx context.Context) (bool, error) { |
| - obs, err := l.syncClaimPhase(ctx) |
| - if err != nil { |
| - return false, err |
| - } |
| - if err := l.requirePhase(obs, provision.LandPublished); err != nil { |
| - return false, err |
| - } |
| - j, err := l.journal() |
| - if err != nil { |
| - return false, err |
| - } |
| - if !j.Done(landStepInstall) { |
| - return false, fmt.Errorf("move %s: published without an install manifest in the journal", l.g.ID) |
| - } |
| - j.Phase = provision.LandPublished |
| - j.Mark(landStepPublished, l.cfg().Now().UTC().Format(time.RFC3339Nano)) |
| - if err := j.Write(l.cfg().DataDir); err != nil { |
| - return false, err |
| - } |
| - ok, err := l.advance(ctx, provision.LandPublished, provision.LandLocal, nil) |
| - if ok { |
| - l.releaseLease() |
| - } |
| - return ok, err |
| -} |
| - |
| -// local (§7.6), each sub-step journalled and skipped on a replay once done, |
| -// in this order (round-2 finding 8): (1) the installation record must be at |
| -// F; (2) an earlier stay's artifacts tagged S' are parked into .parked/ |
| -// through park.Local (which refuses a path tagged with another stamp), or a |
| -// park-local left with nothing to retire is completed; (3) the sealed |
| -// archive, read by exact version and verified, is extracted into this op's |
| -// own staging, <JournalDir>/extract/, and then, inside the host guard, moved |
| -// to the files directory and <host>.env when nothing this landing did not |
| -// write is there (installFiles: the landing never deletes at the live |
| -// paths, final review finding 1; a retry clears only its staging); (4) the |
| -// new paths are registered under F; (5) the timers |
| -// table is the sealed snapshot with its registrations, the carried columns |
| -// are set and the restore-verification entry removed; (6) the prepared |
| -// files manifest, skipped with a log line until C8 exists; (7) the files |
| -// directory is synced and .files-gen written last, so its presence vouches |
| -// for every step before it. |
| -// |
| -// A replay finds local and finishes what the journal does not record; |
| -// nothing here is read by the probe before .files-gen exists. |
| -func (l *landingRun) local_(ctx context.Context) (bool, error) { |
| - cfg := l.cfg() |
| - host, id := l.rec.Host, l.g.ID |
| - obs, err := l.syncClaimPhase(ctx) |
| - if err != nil { |
| - return false, err |
| - } |
| - if err := l.requirePhase(obs, provision.LandLocal); err != nil { |
| - return false, err |
| - } |
| - holder := *obs.Claim.Holder |
| - src := l.rec.Move.Source |
| - if src == nil || src.Files == nil { |
| - return false, fmt.Errorf("move %s: the record has no source baselines", id) |
| - } |
| - seal, err := journalledSeal(obs.Claim) |
| - if err != nil { |
| - return false, err |
| - } |
| - j, err := l.journal() |
| - if err != nil { |
| - return false, err |
| - } |
| - j.Phase = provision.LandLocal |
| - mark := func(step, value string) error { |
| - j.Mark(step, value) |
| - if err := j.Write(cfg.DataDir); err != nil { |
| - return err |
| - } |
| - return l.fault("after-" + step) |
| - } |
| - captured, err := l.capture(holder) |
| - if err != nil { |
| - return false, err |
| - } |
| - if !j.Done(landStepRetired) { |
| - if raw, ok := j.Steps[landStepPrevious]; ok { |
| - earlier, err := strconv.ParseUint(raw, 10, 64) |
| - if err != nil || earlier == 0 { |
| - return false, fmt.Errorf("the journal's previous stamp %q", raw) |
| - } |
| - var chore *hostclaim.Obligation |
| - for i := range obs.Claim.Obligations { |
| - if o := &obs.Claim.Obligations[i]; o.Op == id && o.Kind == "park-local" && o.Box == cfg.Box { |
| - chore = o |
| +// installSource binds the shared installer (movesinstall.go) to this |
| +// landing: the claim follows each record step (syncClaimPhase walks it, |
| +// mirrorPhase writes it), the claim check is the landing op at the phase |
| +// with this box executing, the record check a landing at the phase with no |
| +// rollback reason, the refusal the rollback marker (Plan 4's rollback |
| +// transition replaces markRollback), the phase advance MoveLandingPhase |
| +// stamping landing.installed_db_sha256 at the end of configured, and the |
| +// generation, the hold and the archive all the move id's. |
| +func (l *landingRun) installSource() installSource { |
| + m := l.rec.Move |
| + return installSource{ |
| + OpID: l.g.ID, Host: l.rec.Host, Role: roleTarget, Name: "landing", |
| + Lane: moves.StagingPrefix(l.rec.Host, l.g.ID), Seal: journalledSeal, Baseline: m.Source, |
| + GenerationOp: l.g.ID, HoldOp: l.g.ID, PinInputs: m.Inputs, |
| + Phases: [6]string{provision.LandParking, provision.LandBuilding, provision.LandPublished, |
| + provision.LandLocal, provision.LandConfigured, provision.LandRestored}, |
| + Phase: func(from, to string, st installStamp) func(*provision.Record) error { |
| + var stamp func(*provision.MoveState) |
| + if sha := st.InstalledDBSHA256; sha != "" { |
| + stamp = func(m *provision.MoveState) { |
| + if m.Landing == nil { |
| + m.Landing = &provision.MoveLandingJournal{} |
| + } |
| + m.Landing.InstalledDBSHA256 = sha |
| } |
| } |
| - switch { |
| - case chore != nil && hostclaim.InventoryHasStamp(obs.Claim, cfg.Box, earlier): |
| - if err := l.local.RetireArtifacts(ctx, host, captured, chore.ID, id, earlier); err != nil { |
| - return false, fmt.Errorf("retire the earlier stay's artifacts: %w", err) |
| - } |
| - log.Printf("moves: %s: the earlier stay's artifacts (stamp %d) are parked", host, earlier) |
| - case chore != nil: |
| - if _, err := hostclaim.Update(ctx, l.s, obs, hostclaim.CompleteLocalPark(id, cfg.Box, earlier)); err != nil { |
| - return false, fmt.Errorf("complete the local park: %w", err) |
| - } |
| + return provision.MoveLandingPhase(l.g, from, to, stamp) |
| + }, |
| + Mirror: l.mirrorPhase, |
| + Sync: l.syncClaimPhase, |
| + ClaimCheck: l.landingCheck, |
| + RecordPhase: func(rec *provision.Record) (string, error) { |
| + m := rec.Move |
| + if m == nil || m.ID != l.g.ID || m.Incarnation != l.g.Incarnation || m.State != provision.MoveLanding || m.Reason != "" { |
| + return "", fmt.Errorf("%w: the record is no longer landing %s", errLandingChanged, l.g.ID) |
| } |
| - } |
| - if err := mark(landStepRetired, ""); err != nil { |
| - return false, err |
| - } |
| - } |
| - dir, envPath := cfg.FilesDir(host) |
| - if !j.Done(landStepExtracted) { |
| - outcome, err := l.installFiles(ctx, j, mark, captured, seal, *src.Files, dir, envPath) |
| - switch { |
| - case err != nil: |
| - return false, err |
| - case outcome == restoreMarked: |
| - return true, nil // the record changed: the rollback marker |
| - case outcome == restoreWaiting: |
| - return false, nil |
| - } |
| - } |
| - if !j.Done(landStepInventory) { |
| - paths := []string{host + ".db", host + ".db.gen", host, host + ".env"} |
| - if err := l.local.RecordInventory(ctx, host, captured, "flat", paths, l.landingCheck(provision.LandLocal)); err != nil { |
| - return false, fmt.Errorf("record the inventory: %w", err) |
| - } |
| - if err := mark(landStepInventory, ""); err != nil { |
| - return false, err |
| - } |
| - } |
| - if !j.Done(landStepTimers) { |
| - rows, err := l.sealedTimers(ctx, seal, src.TimersSHA256) |
| - if err != nil { |
| - return false, err |
| - } |
| - if err := cfg.ReplaceTimers(ctx, host, rows); err != nil { |
| - return false, fmt.Errorf("replace the timers: %w", err) |
| - } |
| - if err := mark(landStepTimers, strconv.Itoa(len(rows))); err != nil { |
| - return false, err |
| - } |
| - } |
| - if !j.Done(landStepColumns) { |
| - if err := cfg.SetReplication(host, src.Replication); err != nil { |
| - return false, fmt.Errorf("set replication: %w", err) |
| - } |
| - if err := cfg.SetNextDue(host, src.NextDue); err != nil { |
| - return false, fmt.Errorf("set next_due: %w", err) |
| - } |
| - if err := mark(landStepColumns, ""); err != nil { |
| - return false, err |
| - } |
| - } |
| - if !j.Done(landStepVerify) { |
| - if err := cfg.RemoveVerifyEntry(ctx, l.s, host); err != nil { |
| - return false, fmt.Errorf("remove the restore-verification entry: %w", err) |
| - } |
| - if err := mark(landStepVerify, ""); err != nil { |
| - return false, err |
| - } |
| - } |
| - if !j.Done(landStepFilesManifest) { |
| - // Spec rev 7.22 §7.6 (L17, R19-1): the prepared files manifest under |
| - // files/g-<move id>/ in the location, bound to {location, g-<op>, op, |
| - // F}. Its build (C8, files.BuildPrepared) is not on main; until it |
| - // is, the step is recorded as skipped and the probe's admission |
| - // establishes the files baseline (Plan 3). |
| - log.Printf("moves: %s: %s", host, landingFilesManifestSkipped) |
| - if err := mark(landStepFilesManifest, landingFilesManifestSkipped); err != nil { |
| - return false, err |
| - } |
| - } |
| - if err := l.fault("before-files-gen"); err != nil { |
| - return false, err |
| - } |
| - marker, err := json.Marshal(filesGenMarker{Op: id, Incarnation: holder.Incarnation}) |
| - if err != nil { |
| - return false, err |
| - } |
| - placed, err := journalledStaged(j, landStepFilesPlaced) |
| - if err != nil { |
| - return false, err |
| - } |
| - // The marker goes into the files directory this landing placed, and |
| - // only while the installation is still this landing's (final review, |
| - // finding 2): inside the guard, the directory is the receipt's. |
| - if err := l.guarded(ctx, captured, provision.LandLocal, func() error { |
| - if ok, err := sameIdentity(dir, placed.Files); err != nil || !ok { |
| - return fmt.Errorf("%w: %s is not the files directory this landing placed (%v)", errLandingChanged, dir, err) |
| - } |
| - if err := syncPath(dir); err != nil { |
| - return err |
| - } |
| - return writeFileAtomic(filepath.Join(dir, archive.MarkerName), marker, 0o644) |
| - }); err != nil { |
| - return false, fmt.Errorf("write %s: %w", archive.MarkerName, err) |
| - } |
| - if err := mark(landStepFilesGen, ""); err != nil { |
| - return false, err |
| - } |
| - log.Printf("moves: %s: files, environment, timers and columns installed; %s written", host, archive.MarkerName) |
| - return l.advance(ctx, provision.LandLocal, provision.LandConfigured, nil) |
| -} |
| - |
| -// configured (§7.6, §9.1, §9.2): the held row rendered (RenderHeld, the |
| -// move-bound ack), its bundle and overlay etags required to be the ones |
| -// pinned at readiness (else the rollback marker, "configuration changed |
| -// during the move"); the pinned release live and proven for the held row |
| -// through Plan 1's T7 hook (ruling G4: "waiting for the root stage pass" |
| -// waits, anything else waits until the landing's deadline marks the |
| -// rollback); |
| -// then the install the brief lists under `restored`: the built database's |
| -// sha256 against the manifest, lease.CheckDB, the rename into DBPath (G34), |
| -// <dbPath>.gen through hostbox's marker writer naming the prepared |
| -// generation (ruling G2), and the CAS to restored stamping |
| -// landing.installed_db_sha256. |
| -// |
| -// A replay finds configured and does it all again: the render and the |
| -// release check are idempotent reads and writes of the same bytes; a |
| -// database already at DBPath with the manifest's sha256 is this landing's |
| -// own install, so only the marker is written again. |
| -func (l *landingRun) configured(ctx context.Context) (bool, error) { |
| - cfg := l.cfg() |
| - host, id := l.rec.Host, l.g.ID |
| - obs, err := l.syncClaimPhase(ctx) |
| - if err != nil { |
| - return false, err |
| - } |
| - if err := l.requirePhase(obs, provision.LandConfigured); err != nil { |
| - return false, err |
| - } |
| - in := l.rec.Move.Inputs |
| - if in == nil { |
| - return false, fmt.Errorf("move %s: the record has no pinned inputs", id) |
| - } |
| - rt, err := l.route() |
| - if err != nil { |
| - return false, err |
| - } |
| - hr, err := cfg.RenderHeld(ctx, l.s, rt, id) |
| - if err != nil { |
| - return false, fmt.Errorf("render the held row: %w", err) |
| - } |
| - if !sameBundle(hr.BundleETags, in.BundleETags) || hr.OverlayETag != in.OverlayETag { |
| - log.Printf("moves: %s: the render's etags (%v, overlay %q) are not the pinned ones (%v, overlay %q)", host, hr.BundleETags, hr.OverlayETag, in.BundleETags, in.OverlayETag) |
| - return l.markRollback(ctx, provision.RollbackConfigChanged) |
| - } |
| - if c := cfg.ReleaseReady(ctx, l.s, l.rec, in.Release); !c.OK { |
| - l.w.notice(host+"#release", "moves: %s: release %s is not live and proven for the held row yet (%s); retrying until the landing's deadline", host, in.Release, c.Reason) |
| - return false, nil |
| - } |
| - if err := l.fault("after-release-ready"); err != nil { |
| - return false, err |
| - } |
| - // The render, the release gate and the install can outlast the deadline |
| - // (fix round 1, item 4): it is checked again before the install and |
| - // right before the transition that names it complete. |
| - if l.pastDeadline() { |
| - return l.markRollback(ctx, provision.RollbackLandingDeadline) |
| - } |
| - sha, outcome, err := l.restore(ctx, obs) |
| - if err != nil { |
| - return false, err |
| - } |
| - switch outcome { |
| - case restoreWaiting: |
| - return false, nil |
| - case restoreMarked: |
| - return true, nil // the record changed: the rollback marker, not an install |
| - } |
| - if l.pastDeadline() { |
| - return l.markRollback(ctx, provision.RollbackLandingDeadline) |
| + return m.LandingPhase, nil |
| + }, |
| + Refuse: func(ctx context.Context, why installRefusal) (bool, string, error) { |
| + var reason string |
| + switch why { |
| + case refuseFilesOccupied: |
| + reason = provision.RollbackFilesOccupied |
| + case refuseInstallMismatch: |
| + reason = provision.RollbackInstallMismatch |
| + case refuseConfigChanged: |
| + reason = provision.RollbackConfigChanged |
| + case refuseDeadline: |
| + reason = provision.RollbackLandingDeadline |
| + default: |
| + return false, "", fmt.Errorf("move %s: a landing has no refusal %d", l.g.ID, why) |
| + } |
| + ok, err := l.markRollback(ctx, reason) |
| + if !ok { |
| + return false, "", err |
| + } |
| + return true, reason, err |
| + }, |
| + PastDeadline: l.pastDeadline, |
| } |
| - return l.advance(ctx, provision.LandConfigured, provision.LandRestored, func(m *provision.MoveState) { |
| - if m.Landing == nil { |
| - m.Landing = &provision.MoveLandingJournal{} |
| - } |
| - m.Landing.InstalledDBSHA256 = sha |
| - }) |
| } |
diff --git a/internal/host/moveslandinstall.go b/internal/host/moveslandinstall.go| index 4a1be3cfe..bf3186cb0 100644 |
| --- a/internal/host/moveslandinstall.go |
| +++ b/internal/host/moveslandinstall.go |
| @@ -24,13 +24,14 @@ import ( |
| "github.com/carlosframework/platform/internal/registry" |
| ) |
| -// The landing's install helpers (moves spec §7.6 local, restored): the |
| -// sealed archive read by exact version, verified and extracted into the |
| -// op's staging, then placed at the live paths inside the host guard; the |
| -// sealed timers; the database install from the build scratch with its |
| -// marker, inside the guard; the build again when the journal lost it; and |
| -// the fsync and atomic-write helpers. landingPass and the phases are in |
| -// movesland.go. |
| +// The installer's helpers (moves spec §7.6 local, restored; shared by the |
| +// landing and the dirty return, movesinstall.go): the sealed archive read by |
| +// exact version, verified and extracted into the install's staging, then |
| +// placed at the live paths inside the host guard; the sealed timers; the |
| +// database install from the build scratch with its marker, inside the |
| +// guard; the build again when the journal lost it; and the fsync and |
| +// atomic-write helpers. The phases are in movesinstall.go, the landing's |
| +// checking and pass in movesland.go. |
| // errLandingChanged is a guarded install's refusal when the claim, the |
| // record, the installation record or the destination is no longer this |
| @@ -44,45 +45,46 @@ var errFilesOccupied = errors.New("the files paths hold what this landing did no |
| // capture reads the installation record and requires it at the holder's |
| // incarnation and stamp: the identity every guarded install runs under. |
| -func (l *landingRun) capture(holder hostclaim.Holder) (hostguard.Captured, error) { |
| - captured, err := hostguard.Capture(l.cfg().DataDir, l.rec.Host) |
| +func (ir *installRun) capture(holder hostclaim.Holder) (hostguard.Captured, error) { |
| + captured, err := hostguard.Capture(ir.cfg().DataDir, ir.rec.Host) |
| if err != nil { |
| return captured, fmt.Errorf("read the installation record: %w", err) |
| } |
| if captured.Incarnation != holder.Incarnation || captured.Stamp != holder.Stamp { |
| - return captured, fmt.Errorf("move %s: the installation record is at stamp %d of %s, not this landing's %d", l.g.ID, captured.Stamp, captured.Incarnation, holder.Stamp) |
| + return captured, fmt.Errorf("move %s: the installation record is at stamp %d of %s, not this %s's %d", ir.g.ID, captured.Stamp, captured.Incarnation, ir.src.Name, holder.Stamp) |
| } |
| return captured, nil |
| } |
| -// guarded runs fn inside the host guard for this landing's installation |
| +// guarded runs fn inside the host guard for this install's installation |
| // (final review, finding 2; hostbox ColdRestore.Install's pattern): under |
| // the host's lock, the installation record still active at the captured |
| -// stamp, no fence over it, the claim read fresh and still this landing's op |
| -// at phase with this box executing and the holder at the captured stamp, |
| -// and the record read fresh and still this landing at phase with no |
| -// rollback reason. fn re-checks its destination and makes the install; it |
| -// must not call anything that takes the guard itself (park.Local's |
| -// retirement and inventory), nor wait on the network. |
| -func (l *landingRun) guarded(ctx context.Context, captured hostguard.Captured, phase string, fn func() error) error { |
| +// stamp, no fence over it, the claim read fresh and still passing the |
| +// install's claim check at phase (a landing's op at phase with this box |
| +// executing; a return's restoring/claimed with this box executing) with the |
| +// holder at the captured stamp, and the record read fresh and still this |
| +// install at phase (installSource.RecordPhase: a landing with no rollback |
| +// reason; a dirty return). fn re-checks its destination and makes the |
| +// install; it must not call anything that takes the guard itself |
| +// (park.Local's retirement and inventory), nor wait on the network. |
| +func (ir *installRun) guarded(ctx context.Context, captured hostguard.Captured, phase string, fn func() error) error { |
| check := func(c *hostclaim.Claim) error { |
| - if err := l.landingCheck(phase)(c); err != nil { |
| + if err := ir.src.ClaimCheck(phase)(c); err != nil { |
| return err |
| } |
| if c.Holder == nil || c.Holder.Stamp != captured.Stamp { |
| return fmt.Errorf("%w: the claim's holder is not at the installation stamp %d", errLandingChanged, captured.Stamp) |
| } |
| - rec, err := provision.GetRecord(ctx, l.s, l.rec.Account, l.rec.App, l.rec.Host) |
| + rec, err := provision.GetRecord(ctx, ir.s, ir.rec.Account, ir.rec.App, ir.rec.Host) |
| if err != nil { |
| return fmt.Errorf("re-read the record: %w", err) |
| } |
| - m := rec.Move |
| - if m == nil || m.ID != l.g.ID || m.Incarnation != l.g.Incarnation || m.State != provision.MoveLanding || m.LandingPhase != phase || m.Reason != "" { |
| - return fmt.Errorf("%w: the record is no longer landing %s at %s", errLandingChanged, l.g.ID, phase) |
| + if at, err := ir.src.RecordPhase(rec); err != nil || at != phase { |
| + return fmt.Errorf("%w: the record is no longer the %s of %s at %s", errLandingChanged, ir.src.Name, ir.g.ID, phase) |
| } |
| return nil |
| } |
| - return l.guard.With(ctx, l.rec.Host, captured, check, fn) |
| + return ir.guard.With(ctx, ir.rec.Host, captured, check, fn) |
| } |
| // stagedFiles names the staged extraction by identity (device and inode): |
| @@ -97,7 +99,7 @@ type stagedFiles struct { |
| // stageDir is the op's own extraction staging, <JournalDir>/extract/: the |
| // one place a retry clears (final review, finding 1). |
| -func (l *landingRun) stageDir() string { return filepath.Join(l.journalDir(), "extract") } |
| +func (ir *installRun) stageDir() string { return filepath.Join(ir.journalDir(), "extract") } |
| // journalledStaged reads a stagedFiles journal step. |
| func journalledStaged(j *moves.Journal, step string) (stagedFiles, error) { |
| @@ -122,8 +124,8 @@ func journalledStaged(j *moves.Journal, step string) (stagedFiles, error) { |
| // else there is not the landing's to clear: nothing is moved and the |
| // rollback is marked. The install receipt (`files_placed`) is journalled, |
| // the empty staging removed, and only then `extracted`. |
| -func (l *landingRun) installFiles(ctx context.Context, j *moves.Journal, mark func(step, value string) error, captured hostguard.Captured, seal moves.Seal, want provision.MoveFilesDigest, dir, envPath string) (restoreOutcome, error) { |
| - stage := l.stageDir() |
| +func (ir *installRun) installFiles(ctx context.Context, j *moves.Journal, mark func(step, value string) error, captured hostguard.Captured, seal moves.Seal, want provision.MoveFilesDigest, dir, envPath string) (restoreOutcome, error) { |
| + stage := ir.stageDir() |
| stFiles, stEnv := filepath.Join(stage, "files"), filepath.Join(stage, "env") |
| var staged stagedFiles |
| if j.Done(landStepStaged) { |
| @@ -140,7 +142,7 @@ func (l *landingRun) installFiles(ctx context.Context, j *moves.Journal, mark fu |
| if err := os.MkdirAll(stage, 0o700); err != nil { |
| return restoreWaiting, err |
| } |
| - if err := l.extractArchive(ctx, seal, want, stFiles, stEnv); err != nil { |
| + if err := ir.extractArchive(ctx, seal, want, stFiles, stEnv); err != nil { |
| return restoreWaiting, fmt.Errorf("extract the archive: %w", err) |
| } |
| var err error |
| @@ -162,12 +164,12 @@ func (l *landingRun) installFiles(ctx context.Context, j *moves.Journal, mark fu |
| return restoreWaiting, err |
| } |
| } |
| - err := l.guarded(ctx, captured, provision.LandLocal, func() error { |
| - return l.placeStaged(staged, stFiles, stEnv, dir, envPath) |
| + err := ir.guarded(ctx, captured, ir.src.Phases[3], func() error { |
| + return ir.placeStaged(staged, stFiles, stEnv, dir, envPath) |
| }) |
| if errors.Is(err, errFilesOccupied) { |
| - log.Printf("moves: %s: %v; nothing was moved or removed", l.rec.Host, err) |
| - ok, err := l.markRollback(ctx, provision.RollbackFilesOccupied) |
| + log.Printf("moves: %s: %v; nothing was moved or removed", ir.rec.Host, err) |
| + ok, err := ir.refuse(ctx, refuseFilesOccupied) |
| if err != nil || !ok { |
| return restoreWaiting, err |
| } |
| @@ -194,7 +196,7 @@ func (l *landingRun) installFiles(ctx context.Context, j *moves.Journal, mark fu |
| // absent or an empty directory, and <host>.env this landing's env already, |
| // or absent. An empty directory gives its mode to the tree and is replaced, |
| // as Extract has always done. |
| -func (l *landingRun) placeStaged(staged stagedFiles, stFiles, stEnv, dir, envPath string) error { |
| +func (ir *installRun) placeStaged(staged stagedFiles, stFiles, stEnv, dir, envPath string) error { |
| filesDone, err := sameIdentity(dir, staged.Files) |
| if err != nil { |
| return err |
| @@ -246,7 +248,7 @@ func (l *landingRun) placeStaged(staged stagedFiles, stFiles, stEnv, dir, envPat |
| if err := syncPath(filepath.Dir(dir)); err != nil { |
| return err |
| } |
| - if err := l.fault("after-files-rename"); err != nil { |
| + if err := ir.fault("after-files-rename"); err != nil { |
| return err |
| } |
| } |
| @@ -344,8 +346,8 @@ type filesGenMarker struct { |
| // record's digests (archive.Verify: the sealed digest, every entry), and |
| // extracts it into dir and envPath (archive.Open under the box's secrets |
| // key, then archive.Extract, which admits nothing it did not read whole). |
| -func (l *landingRun) extractArchive(ctx context.Context, seal moves.Seal, want provision.MoveFilesDigest, dir, envPath string) error { |
| - cfg := l.cfg() |
| +func (ir *installRun) extractArchive(ctx context.Context, seal moves.Seal, want provision.MoveFilesDigest, dir, envPath string) error { |
| + cfg := ir.cfg() |
| var sealedObj, manifestObj *moves.SealedObject |
| for i := range seal.Files { |
| switch o := &seal.Files[i]; { |
| @@ -358,7 +360,7 @@ func (l *landingRun) extractArchive(ctx context.Context, seal moves.Seal, want p |
| if sealedObj == nil || manifestObj == nil { |
| return errors.New("the seal does not name the archive and its manifest") |
| } |
| - vs, err := l.versioned(ctx) |
| + vs, err := ir.versioned(ctx) |
| if err != nil { |
| return err |
| } |
| @@ -370,13 +372,13 @@ func (l *landingRun) extractArchive(ctx context.Context, seal moves.Seal, want p |
| if err := json.Unmarshal(raw, &m); err != nil { |
| return fmt.Errorf("the archive manifest: %w", err) |
| } |
| - if m.Op != l.g.ID || m.Incarnation != l.rec.Move.Incarnation || m.Plain != want.Plain || m.Sealed != want.Sealed { |
| - return fmt.Errorf("the archive manifest is for %s/%s (%s, %s), not this move's %s (%s, %s)", m.Op, m.Incarnation, m.Plain, m.Sealed, l.g.ID, want.Plain, want.Sealed) |
| + if m.Op != ir.src.OpID || m.Incarnation != ir.rec.Move.Incarnation || m.Plain != want.Plain || m.Sealed != want.Sealed { |
| + return fmt.Errorf("the archive manifest is for %s/%s (%s, %s), not this move's %s (%s, %s)", m.Op, m.Incarnation, m.Plain, m.Sealed, ir.src.OpID, want.Plain, want.Sealed) |
| } |
| - if err := os.MkdirAll(l.journalDir(), 0o700); err != nil { |
| + if err := os.MkdirAll(ir.journalDir(), 0o700); err != nil { |
| return err |
| } |
| - sealedPath := filepath.Join(l.journalDir(), "archive.sealed") |
| + sealedPath := filepath.Join(ir.journalDir(), "archive.sealed") |
| if err := os.Remove(sealedPath); err != nil && !errors.Is(err, os.ErrNotExist) { |
| return err |
| } |
| @@ -389,7 +391,7 @@ func (l *landingRun) extractArchive(ctx context.Context, seal moves.Seal, want p |
| return err |
| } |
| defer f.Close() |
| - bind := appconfig.Binding{Account: l.rec.Account, App: l.rec.App} |
| + bind := appconfig.Binding{Account: ir.rec.Account, App: ir.rec.App} |
| if err := archive.Verify(f, m, cfg.SecretsPriv, bind); err != nil { |
| return fmt.Errorf("verify the archive: %w", err) |
| } |
| @@ -418,12 +420,12 @@ func openAndExtract(f *os.File, m archive.Manifest, priv *ecdh.PrivateKey, bind |
| // sealedTimers reads the sealed timers snapshot by exact version and checks |
| // its digest against the record's; every row is this host's. |
| -func (l *landingRun) sealedTimers(ctx context.Context, seal moves.Seal, wantDigest string) ([]registry.Timer, error) { |
| - cfg := l.cfg() |
| +func (ir *installRun) sealedTimers(ctx context.Context, seal moves.Seal, wantDigest string) ([]registry.Timer, error) { |
| + cfg := ir.cfg() |
| if len(seal.Timers) != 1 { |
| return nil, fmt.Errorf("the seal names %d timers objects, want 1", len(seal.Timers)) |
| } |
| - vs, err := l.versioned(ctx) |
| + vs, err := ir.versioned(ctx) |
| if err != nil { |
| return nil, err |
| } |
| @@ -435,15 +437,15 @@ func (l *landingRun) sealedTimers(ctx context.Context, seal moves.Seal, wantDige |
| if err != nil { |
| return nil, err |
| } |
| - if snap.Host != l.rec.Host { |
| - return nil, fmt.Errorf("the timers snapshot is %s's, not %s's", snap.Host, l.rec.Host) |
| + if snap.Host != ir.rec.Host { |
| + return nil, fmt.Errorf("the timers snapshot is %s's, not %s's", snap.Host, ir.rec.Host) |
| } |
| if got := moves.TimersDigest(snap.Rows); got != wantDigest { |
| return nil, fmt.Errorf("the timers snapshot digests to %s, the record says %s", got, wantDigest) |
| } |
| for _, r := range snap.Rows { |
| - if r.Host != l.rec.Host { |
| - return nil, fmt.Errorf("timer %q is %s's, not %s's", r.Name, r.Host, l.rec.Host) |
| + if r.Host != ir.rec.Host { |
| + return nil, fmt.Errorf("timer %q is %s's, not %s's", r.Name, r.Host, ir.rec.Host) |
| } |
| } |
| return snap.Rows, nil |
| @@ -488,16 +490,16 @@ const ( |
| // DBPath that is not this build, a marker naming another generation, or a |
| // WAL beside an absent database is a successor's or someone else's, and |
| // is never replaced. |
| -func (l *landingRun) restore(ctx context.Context, obs hostclaim.Observed) (sha string, outcome restoreOutcome, err error) { |
| - cfg := l.cfg() |
| - host, id := l.rec.Host, l.g.ID |
| - m, waiting, err := l.installManifest(ctx, obs) |
| +func (ir *installRun) restore(ctx context.Context, obs hostclaim.Observed) (sha string, outcome restoreOutcome, err error) { |
| + cfg := ir.cfg() |
| + host, id := ir.rec.Host, ir.src.GenerationOp |
| + m, waiting, err := ir.installManifest(ctx, obs) |
| if err != nil || waiting { |
| return "", restoreWaiting, err |
| } |
| want := "sha256:" + m.DBSHA256 |
| - dbPath := l.dbPath() |
| - captured, err := l.capture(*obs.Claim.Holder) |
| + dbPath := ir.dbPath() |
| + captured, err := ir.capture(*obs.Claim.Holder) |
| if err != nil { |
| return "", restoreWaiting, err |
| } |
| @@ -505,7 +507,7 @@ func (l *landingRun) restore(ctx context.Context, obs hostclaim.Observed) (sha s |
| if err != nil { |
| return "", restoreWaiting, err |
| } |
| - built := filepath.Join(l.buildDir(), host+".db") |
| + built := filepath.Join(ir.buildDir(), host+".db") |
| if !present { |
| if ok, err := regularFile(built); err != nil { |
| return "", restoreWaiting, err |
| @@ -513,11 +515,11 @@ func (l *landingRun) restore(ctx context.Context, obs hostclaim.Observed) (sha s |
| // The scratch was lost (a rebuilt data directory): build again |
| // from the pinned set, which reproduces the same bytes, under |
| // the location's lease. |
| - if _, waiting, err := l.rebuild(ctx, obs); err != nil || waiting { |
| + if _, waiting, err := ir.rebuild(ctx, obs); err != nil || waiting { |
| return "", restoreWaiting, err |
| } |
| } |
| - if err := l.fault("before-db-verify"); err != nil { |
| + if err := ir.fault("before-db-verify"); err != nil { |
| return "", restoreWaiting, err |
| } |
| got, err := fileSHA256(built) |
| @@ -526,17 +528,17 @@ func (l *landingRun) restore(ctx context.Context, obs hostclaim.Observed) (sha s |
| } |
| if got != want { |
| log.Printf("moves: %s: the built database is %s, the manifest says %s", host, got, want) |
| - return l.refuseInstall(ctx) |
| + return ir.refuseInstall(ctx) |
| } |
| if err := lease.CheckDB(built); err != nil { |
| log.Printf("moves: %s: the built database does not pass its check: %v", host, err) |
| - return l.refuseInstall(ctx) |
| + return ir.refuseInstall(ctx) |
| } |
| for _, side := range []string{"-wal", "-shm"} { |
| os.Remove(built + side) |
| } |
| } |
| - err = l.guarded(ctx, captured, provision.LandConfigured, func() error { |
| + err = ir.guarded(ctx, captured, ir.src.Phases[4], func() error { |
| // Every destination is Lstat-checked, never read through: a symlink |
| // or any other non-regular file at <dbPath>.gen, DBPath or the WAL |
| // paths, dangling or not, is not this landing's and is never |
| @@ -586,7 +588,7 @@ func (l *landingRun) restore(ctx context.Context, obs hostclaim.Observed) (sha s |
| return err |
| } |
| } |
| - if err := l.fault("after-db-rename"); err != nil { |
| + if err := ir.fault("after-db-rename"); err != nil { |
| return err |
| } |
| if err := cfg.WriteGen(dbPath+".gen", id); err != nil { |
| @@ -601,11 +603,11 @@ func (l *landingRun) restore(ctx context.Context, obs hostclaim.Observed) (sha s |
| return want, restoreInstalled, nil |
| } |
| -// refuseInstall marks the rollback for a build that does not verify. A |
| -// marker that landed is restoreMarked; one refused (the record moved on) is |
| -// restoreWaiting, and nothing is installed either way. |
| -func (l *landingRun) refuseInstall(ctx context.Context) (string, restoreOutcome, error) { |
| - ok, err := l.markRollback(ctx, provision.RollbackInstallMismatch) |
| +// refuseInstall writes the install's refusal for a build that does not |
| +// verify. A refusal that landed is restoreMarked; one refused (the record |
| +// moved on) is restoreWaiting, and nothing is installed either way. |
| +func (ir *installRun) refuseInstall(ctx context.Context) (string, restoreOutcome, error) { |
| + ok, err := ir.refuse(ctx, refuseInstallMismatch) |
| if err != nil || !ok { |
| return "", restoreWaiting, err |
| } |
| @@ -614,16 +616,16 @@ func (l *landingRun) refuseInstall(ctx context.Context) (string, restoreOutcome, |
| // installManifest is the journal's, or the build's again when the journal |
| // lost it. waiting is true when the rebuild could not take the lease. |
| -func (l *landingRun) installManifest(ctx context.Context, obs hostclaim.Observed) (moves.InstallManifest, bool, error) { |
| - j, err := l.journal() |
| +func (ir *installRun) installManifest(ctx context.Context, obs hostclaim.Observed) (moves.InstallManifest, bool, error) { |
| + j, err := ir.journal() |
| if err != nil { |
| return moves.InstallManifest{}, false, err |
| } |
| var m moves.InstallManifest |
| - if raw, ok := j.Steps[landStepInstall]; ok && json.Unmarshal([]byte(raw), &m) == nil && m.Op == l.g.ID && m.Host == l.rec.Host && m.DBSHA256 != "" { |
| + if raw, ok := j.Steps[landStepInstall]; ok && json.Unmarshal([]byte(raw), &m) == nil && m.Op == ir.src.GenerationOp && m.Host == ir.rec.Host && m.DBSHA256 != "" { |
| return m, false, nil |
| } |
| - return l.rebuild(ctx, obs) |
| + return ir.rebuild(ctx, obs) |
| } |
| // rebuild runs BuildGeneration again over the seal: a replay that reuses |
| @@ -632,31 +634,31 @@ func (l *landingRun) installManifest(ctx context.Context, obs hostclaim.Observed |
| // generation request under it, checks it again before recording the result |
| // and releases it after (fix round 1, item 3). waiting is true when another |
| // holder has the lease; nothing is written then. |
| -func (l *landingRun) rebuild(ctx context.Context, obs hostclaim.Observed) (moves.InstallManifest, bool, error) { |
| - cfg := l.cfg() |
| - seal, err := journalledSeal(obs.Claim) |
| +func (ir *installRun) rebuild(ctx context.Context, obs hostclaim.Observed) (moves.InstallManifest, bool, error) { |
| + cfg := ir.cfg() |
| + seal, err := ir.seal(obs.Claim) |
| if err != nil { |
| return moves.InstallManifest{}, false, err |
| } |
| - vs, err := l.versioned(ctx) |
| + vs, err := ir.versioned(ctx) |
| if err != nil { |
| return moves.InstallManifest{}, false, err |
| } |
| holder := *obs.Claim.Holder |
| - held, waiting, err := l.lease(ctx, holder) |
| + held, waiting, err := ir.lease(ctx, holder) |
| if waiting || err != nil { |
| return moves.InstallManifest{}, waiting, err |
| } |
| - defer l.releaseLease() |
| + defer ir.releaseLease() |
| m, err := moves.BuildGeneration(leaseContext(ctx, held), moves.BuildInputs{Store: leasedStore{VersionedStore: vs, check: held.Check}, Bucket: cfg.OwnBucket, Source: seal.Replica, N: seal.N, |
| - GenerationPrefix: moves.GenerationPrefix(holder.Location, l.g.ID), Op: l.g.ID, Host: l.rec.Host, WorkDir: l.buildDir(), Now: cfg.Now}) |
| + GenerationPrefix: moves.GenerationPrefix(holder.Location, ir.src.GenerationOp), Op: ir.src.GenerationOp, Host: ir.rec.Host, WorkDir: ir.buildDir(), Now: cfg.Now}) |
| if err != nil { |
| return moves.InstallManifest{}, false, fmt.Errorf("rebuild the generation: %w", err) |
| } |
| if err := held.Check(ctx); err != nil { |
| return moves.InstallManifest{}, false, fmt.Errorf("the location's lease after the rebuild: %w", err) |
| } |
| - j, err := l.journal() |
| + j, err := ir.journal() |
| if err != nil { |
| return m, false, err |
| } |
diff --git a/internal/host/moveslandpark.go b/internal/host/moveslandpark.go| index b57c0ef15..9c99185a9 100644 |
| --- a/internal/host/moveslandpark.go |
| +++ b/internal/host/moveslandpark.go |
| @@ -15,43 +15,73 @@ import ( |
| "github.com/carlosframework/platform/internal/provision" |
| ) |
| -// The landing's park and lease adapters (moves spec §6.4, §7.6; ruling |
| -// G17): the park engine over this box's own bucket for the residue parks, |
| -// the stand-in view and authorizer the tests use until Plan 5's providers, |
| -// the generation classifier they share, and the own-location lease the |
| -// build holds. landingPass and the phases are in movesland.go. |
| - |
| -// authorizeChore is park.Local.Authorize's stand-in until Plan 5: a chore is |
| -// run only when it is this landing's own and the claim still names this |
| -// landing with this box executing. |
| -func (l *landingRun) authorizeChore(_ context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error { |
| - if c == nil || o.Op != l.g.ID || c.Op.ID != l.g.ID || c.Op.Kind != hostclaim.KindLanding || c.Op.Exec != l.cfg().Box { |
| - return fmt.Errorf("%w: not this landing's chore", park.ErrUnsafe) |
| +// The installer's park and lease adapters (moves spec §6.4, §7.6; ruling |
| +// G17), shared by the landing and the dirty return: the park engine over |
| +// this box's own bucket for the residue parks, the stand-in view and |
| +// authorizer the tests use until Plan 5's providers, the generation |
| +// classifier they share, and the own-location lease the build holds. The |
| +// phases are in movesinstall.go, the landing's pass in movesland.go. |
| + |
| +// authorizeMoveChore is park.Local.Authorize's and park.Engine.Authorize's |
| +// stand-in rule until Plan 5: a chore is run only when it is the op's own |
| +// and the claim still names the op, at the install's kind, with this box |
| +// executing. |
| +func authorizeMoveChore(c *hostclaim.Claim, o hostclaim.Obligation, opID, kind, box string) error { |
| + if c == nil || o.Op != opID || c.Op.ID != opID || c.Op.Kind != kind || c.Op.Exec != box { |
| + return fmt.Errorf("%w: not this install's chore", park.ErrUnsafe) |
| } |
| return nil |
| } |
| +// authorizeChore is park.Local.Authorize's stand-in for this install. |
| +func (ir *installRun) authorizeChore(_ context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error { |
| + return authorizeMoveChore(c, o, ir.g.ID, ir.src.kind(), ir.cfg().Box) |
| +} |
| + |
| // authorizeObligation is park.Engine.Authorize's stand-in (ruling G17): the |
| // same rule over an observation. |
| -func (l *landingRun) authorizeObligation(ctx context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error { |
| +func (ir *installRun) authorizeObligation(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error { |
| if obs.Absent { |
| return park.ErrUnsafe |
| } |
| - return l.authorizeChore(ctx, obs.Claim, o) |
| + return authorizeMoveChore(obs.Claim, o, ir.g.ID, ir.src.kind(), ir.cfg().Box) |
| } |
| -// landingView is park.Engine.View's stand-in (ruling G17): the location's |
| -// objects classified by key, and the source's region authoritative, which |
| -// §5.1's table says it is for a landing before the target's probe is |
| -// admitted (every phase here). Plan 5's provider replaces it; a test may set |
| -// cfg.ParkView to the view a later state has. |
| -func (l *landingRun) landingView(_ context.Context, obs hostclaim.Observed) (park.View, error) { |
| - if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil { |
| - return park.View{}, park.ErrUnsafe |
| +// authorizeObligation is the landing's engine stand-in, the installer's rule |
| +// for a landing. |
| +func (l *landingRun) authorizeObligation(_ context.Context, obs hostclaim.Observed, o hostclaim.Obligation) error { |
| + if obs.Absent { |
| + return park.ErrUnsafe |
| + } |
| + return authorizeMoveChore(obs.Claim, o, l.g.ID, hostclaim.KindLanding, l.cfg().Box) |
| +} |
| + |
| +// standInView is park.Engine.View's stand-in (ruling G17): the location's |
| +// objects classified by key, seen from region, and the other side of the |
| +// move authoritative, which §5.1's table says it is before this side's |
| +// probe is admitted (every install phase). role names this side (ruling |
| +// J24): a landing (roleTarget) works while the source's region, From, is |
| +// authoritative; a dirty return (roleSource) while the target's, To, is. |
| +// Plan 5's provider replaces it; a test may set cfg.ParkView to the view a |
| +// later state has. |
| +func standInView(region, role string, m *provision.MoveState) func(context.Context, hostclaim.Observed) (park.View, error) { |
| + return func(_ context.Context, obs hostclaim.Observed) (park.View, error) { |
| + if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || m == nil { |
| + return park.View{}, park.ErrUnsafe |
| + } |
| + var authoritative string |
| + switch role { |
| + case roleTarget: |
| + authoritative = m.From |
| + case roleSource: |
| + authoritative = m.To |
| + default: |
| + return park.View{}, park.ErrUnsafe |
| + } |
| + location := strings.TrimSuffix(obs.Claim.Holder.Location, "/") |
| + return park.View{ClaimETag: obs.ETag, Region: region, AuthoritativeRegion: authoritative, |
| + GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil |
| } |
| - location := strings.TrimSuffix(obs.Claim.Holder.Location, "/") |
| - return park.View{ClaimETag: obs.ETag, Region: l.cfg().Region, AuthoritativeRegion: l.rec.Move.From, |
| - GenerationOf: func(r park.Ref) (string, bool) { return landingGenerationOf(location, r.Key) }}, nil |
| } |
| // landingMarkerGeneration is what landingGenerationOf answers for the |
| @@ -104,33 +134,33 @@ func isHexString(s string) bool { |
| // engine is the park engine over this box's own bucket: the wired leases, |
| // and the view and authorizer Plan 5 supplies, or the stand-ins. |
| -func (l *landingRun) engine(vs lease.VersionedStore) park.Engine { |
| - cfg := l.cfg() |
| +func (ir *installRun) engine(vs lease.VersionedStore) park.Engine { |
| + cfg := ir.cfg() |
| view, auth := cfg.ParkView, cfg.ParkAuthorize |
| if view == nil { |
| - view = l.landingView |
| + view = standInView(cfg.Region, ir.src.Role, ir.rec.Move) |
| } |
| if auth == nil { |
| - auth = l.authorizeObligation |
| + auth = ir.authorizeObligation |
| } |
| - return park.Engine{Store: vs, Claims: l.s, Bucket: cfg.OwnBucket, Region: cfg.Region, Leases: cfg.ParkLeases, |
| + return park.Engine{Store: vs, Claims: ir.s, Bucket: cfg.OwnBucket, Region: cfg.Region, Leases: cfg.ParkLeases, |
| Now: cfg.Now, NewID: obligationID, View: view, Authorize: auth, After: cfg.parkFault} |
| } |
| // lease is the own-location lease the build holds (§7.6 checking: "hold it |
| // through published"), kept in the worker's per-host map across passes and |
| // re-acquired after a restart. waiting is true when another holder has it. |
| -func (l *landingRun) lease(ctx context.Context, holder hostclaim.Holder) (held park.HeldLease, waiting bool, err error) { |
| - cfg := l.cfg() |
| - host := l.rec.Host |
| - l.w.mu.Lock() |
| - held = l.w.landLeases[host] |
| - l.w.mu.Unlock() |
| +func (ir *installRun) lease(ctx context.Context, holder hostclaim.Holder) (held park.HeldLease, waiting bool, err error) { |
| + cfg := ir.cfg() |
| + host := ir.rec.Host |
| + ir.w.mu.Lock() |
| + held = ir.w.landLeases[host] |
| + ir.w.mu.Unlock() |
| if held != nil { |
| if err := held.Check(ctx); err == nil { |
| return held, false, nil |
| } |
| - l.releaseLease() |
| + ir.releaseLease() |
| } |
| held, err = cfg.OwnLease(ctx, cfg.OwnBucket, strings.TrimSuffix(holder.Location, "/")) |
| if err != nil { |
| @@ -141,22 +171,26 @@ func (l *landingRun) lease(ctx context.Context, holder hostclaim.Holder) (held p |
| } |
| return nil, false, fmt.Errorf("the location's lease: %w", err) |
| } |
| - l.w.mu.Lock() |
| - if l.w.landLeases == nil { |
| - l.w.landLeases = map[string]park.HeldLease{} |
| + ir.w.mu.Lock() |
| + if ir.w.landLeases == nil { |
| + ir.w.landLeases = map[string]park.HeldLease{} |
| } |
| - l.w.landLeases[host] = held |
| - l.w.mu.Unlock() |
| + ir.w.landLeases[host] = held |
| + ir.w.mu.Unlock() |
| return held, false, nil |
| } |
| // releaseLease releases the host's own-location lease, if held. |
| -func (l *landingRun) releaseLease() { |
| - host := l.rec.Host |
| - l.w.mu.Lock() |
| - held := l.w.landLeases[host] |
| - delete(l.w.landLeases, host) |
| - l.w.mu.Unlock() |
| +func (ir *installRun) releaseLease() { releaseOwnLease(ir.w, ir.rec.Host) } |
| + |
| +// releaseOwnLease releases host's own-location lease on w, if held. The |
| +// landing's pass settles it at its end (settleLease); a return's caller |
| +// does the same once its install call returns. |
| +func releaseOwnLease(w *movesWorker, host string) { |
| + w.mu.Lock() |
| + held := w.landLeases[host] |
| + delete(w.landLeases, host) |
| + w.mu.Unlock() |
| if held == nil { |
| return |
| } |
| @@ -172,7 +206,7 @@ func (l *landingRun) releaseLease() { |
| func (l *landingRun) settleLease() { |
| m := l.rec.Move |
| if m == nil || m.State != provision.MoveLanding || m.Reason != "" || !landingBefore(m.LandingPhase, provision.LandLocal) { |
| - l.releaseLease() |
| + releaseOwnLease(l.w, l.rec.Host) |
| } |
| } |