amadan
offline-move: console doors fix round 1 (immutable seal, literal routes, start lock, failed flag, left paths)
The seal is written once, create-only, at control/offline-moves/<host>/<id>/seal.json and source.evidence names it by hash, so the sealer's per-pass report can change without breaking flip. The doors are literal cliRoute registrations (route-gate tests green). Start takes a create-only active lock instead of listing every record. The status view carries failed and the paths a source left.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
6 files changed,
+644
−99
internal/console/apiinstancemove_offline.go+213 −57internal/console/apiinstancemove_offline_test.go+293 −36internal/console/authcli.go+7 −1internal/console/svcroutes_test.go+15 −5internal/moves/offline.go+64 −0internal/moves/offline_test.go+52 −0
diff --git a/internal/console/apiinstancemove_offline.go b/internal/console/apiinstancemove_offline.go| index 32354ac35..033ceb635 100644 |
| --- a/internal/console/apiinstancemove_offline.go |
| +++ b/internal/console/apiinstancemove_offline.go |
| @@ -21,8 +21,8 @@ import ( |
| "github.com/carlosframework/platform/internal/hostops" |
| "github.com/carlosframework/platform/internal/moves" |
| "github.com/carlosframework/platform/internal/provision" |
| + "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/status" |
| - "github.com/carlosframework/platform/internal/svcrole" |
| ) |
| // The offline operator move's doors (docs/superpowers/plans/2026-10-20- |
| @@ -57,7 +57,7 @@ const offlineMoveCapability = "offline-move-1" |
| const offlineReportFresh = 2 * time.Minute |
| // Every sentence a door speaks has its row in the copy record (MV-O20 to |
| -// MV-O31, MV-O40, MV-O41) or is an existing approved one, written out whole |
| +// MV-O31, MV-O50, MV-O51) or is an existing approved one, written out whole |
| // where it is used so the copy gate reads it. |
| // offlineName is a region code as the operator reads it: the home's |
| @@ -163,8 +163,13 @@ type offlineReport struct { |
| raw []byte |
| } |
| -// offlineEvidence is every report of one move, as read. |
| -type offlineEvidence struct{ reports []offlineReport } |
| +// offlineEvidence is every report of one move, as read, and its seal: the |
| +// immutable record of what the sealer decided (moves.OfflineSealKey), nil |
| +// when absent or not a valid seal of this move. |
| +type offlineEvidence struct { |
| + reports []offlineReport |
| + seal *offlineReport |
| +} |
| func (e offlineEvidence) get(role, box string) (offlineReport, bool) { |
| for _, r := range e.reports { |
| @@ -199,7 +204,11 @@ func (s *Server) readOfflineEvidence(ctx context.Context, host, id string) (offl |
| return offlineEvidence{}, err |
| } |
| var ev offlineEvidence |
| + sealKey := moves.OfflineSealKey(host, id) |
| for _, key := range keys { |
| + if key == sealKey { |
| + continue // read below, as the seal and not as a report |
| + } |
| raw, _, err := s.cfg.Releases.Get(ctx, key) |
| if errors.Is(err, os.ErrNotExist) { |
| continue |
| @@ -221,6 +230,19 @@ func (s *Server) readOfflineEvidence(ctx context.Context, host, id string) (offl |
| } |
| return ev.reports[i].Box < ev.reports[j].Box |
| }) |
| + raw, _, err := s.cfg.Releases.Get(ctx, sealKey) |
| + switch { |
| + case errors.Is(err, os.ErrNotExist): |
| + case err != nil: |
| + return offlineEvidence{}, err |
| + default: |
| + var rep moves.OfflineReport |
| + if err := json.Unmarshal(raw, &rep); err != nil || moves.ValidateOfflineSeal(rep, host, id) != nil { |
| + log.Printf("console: offline move %s: ignoring %s: not the seal of this move", host, sealKey) |
| + } else { |
| + ev.seal = &offlineReport{OfflineReport: rep, raw: raw} |
| + } |
| + } |
| return ev, nil |
| } |
| @@ -266,10 +288,12 @@ type offlineFleet struct { |
| func (s *Server) readOfflineFleet(ctx context.Context) (offlineFleet, bool) { |
| snap, err := s.liveEdges(ctx) |
| if err != nil || !snap.EvidenceComplete() { |
| + log.Printf("console: offline move: the edges files are not complete evidence (error %v)", err) |
| return offlineFleet{}, false |
| } |
| boxes, complete, err := status.LoadBoxesComplete(ctx, s.cfg.Releases) |
| if err != nil || !complete { |
| + log.Printf("console: offline move: the boxes' status reports are not complete evidence (complete %v, error %v)", complete, err) |
| return offlineFleet{}, false |
| } |
| f := offlineFleet{snap: snap, metrics: map[string]*status.BoxMetrics{}} |
| @@ -458,18 +482,6 @@ func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Re |
| writeRefusal(w, ref) |
| return |
| } |
| - all, unreadable, err := provision.ListAllRecordsStrict(ctx, s.cfg.Releases) |
| - if err != nil || len(unreadable) > 0 { |
| - log.Printf("console: offline move %s: listing records: %v (%d unreadable)", host, err, len(unreadable)) |
| - jsonError(w, http.StatusBadGateway, "could not read the instance") |
| - return |
| - } |
| - for _, o := range all { |
| - if o.Host != host && o.Move.Offline() { |
| - jsonError(w, http.StatusConflict, fmt.Sprintf("%s is moving offline. move one instance at a time", o.Host)) |
| - return |
| - } |
| - } |
| var req struct { |
| Region string `json:"region"` |
| } |
| @@ -501,9 +513,10 @@ func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Re |
| jsonError(w, http.StatusConflict, "the fleet's edges files could not all be read. try again") |
| return |
| } |
| - if to == "" { |
| + if to == "" || rec.Region == "" { |
| // A box that sets CARLOS_REGION to the home's name is home to this |
| - // console and not to itself, so it cannot take the move. |
| + // console and not to itself, so it cannot take a move into home, and |
| + // out of home the placement it publishes is not the record's. |
| for _, f := range fleet.snap.Files { |
| if f.Placement != "" && homePlacementCode(s.moveHomeName(), f.Placement) == "" { |
| jsonError(w, http.StatusConflict, f.Box+" sets CARLOS_REGION, so it cannot take a move into the home region") |
| @@ -527,25 +540,125 @@ func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Re |
| jsonError(w, http.StatusConflict, offlineConflict(err)) |
| return |
| } |
| - s.offlineAdvance(w, r, d, start) |
| + // One offline move at a time: the lock is taken before the record is |
| + // written, and released if the write fails. |
| + if ref := s.offlineTakeLock(ctx, d, id); ref != nil { |
| + writeRefusal(w, ref) |
| + return |
| + } |
| + got, ok := s.offlineWrite(w, r, d, start) |
| + if !ok { |
| + s.offlineReleaseLock(ctx, host, id) |
| + return |
| + } |
| + writeJSON(w, http.StatusAccepted, s.offlineViewOf(ctx, got)) |
| } |
| -// offlineAdvance applies one transition and answers 202 with the status |
| -// view of the record it left. |
| -func (s *Server) offlineAdvance(w http.ResponseWriter, r *http.Request, d offlineDoor, mut func(*provision.Record) error) { |
| +// offlineWrite applies one transition and bumps the serial, or answers the |
| +// refusal itself. When the move it leaves is cleared (finish, or the finish |
| +// that ends an abort) it releases the lock. |
| +func (s *Server) offlineWrite(w http.ResponseWriter, r *http.Request, d offlineDoor, mut func(*provision.Record) error) (*provision.Record, bool) { |
| ctx := r.Context() |
| rec, err := provision.SetRecordMove(ctx, s.cfg.Releases, d.acct.Sqid, d.app, d.host, mut) |
| if errors.Is(err, provision.ErrMoveConflict) { |
| jsonError(w, http.StatusConflict, offlineConflict(err)) |
| - return |
| + return nil, false |
| } |
| if err != nil { |
| log.Printf("console: offline move %s/%s/%s: %v", d.acct.Sqid, d.app, d.host, err) |
| jsonError(w, http.StatusBadGateway, "could not write the move. try again") |
| - return |
| + return nil, false |
| } |
| s.bumpSerial(ctx) |
| - writeJSON(w, http.StatusAccepted, s.offlineViewOf(ctx, rec)) |
| + if rec.Move == nil && d.rec.Move != nil { |
| + s.offlineReleaseLock(ctx, d.host, d.rec.Move.ID) |
| + } |
| + return rec, true |
| +} |
| + |
| +// offlineAdvance applies one transition and answers 202 with the status |
| +// view of the record it left. |
| +func (s *Server) offlineAdvance(w http.ResponseWriter, r *http.Request, d offlineDoor, mut func(*provision.Record) error) { |
| + if rec, ok := s.offlineWrite(w, r, d, mut); ok { |
| + writeJSON(w, http.StatusAccepted, s.offlineViewOf(r.Context(), rec)) |
| + } |
| +} |
| + |
| +// offlineLockGrace is how long a lock is believed whatever its record says: |
| +// the start door takes the lock, then writes the record. |
| +const offlineLockGrace = 2 * time.Minute |
| + |
| +// offlineTakeLock takes control/offline-moves/active.json for this host and |
| +// move, create-only. A lock already there is live while the record it names |
| +// still carries that move, or is younger than offlineLockGrace (MV-O30 naming |
| +// its host); otherwise it is stale (a crashed start, a failed release) and is |
| +// replaced under its etag. A |
| +// lock that cannot be read is not guessed at. |
| +func (s *Server) offlineTakeLock(ctx context.Context, d offlineDoor, id string) *offlineRefusal { |
| + unreadable := &offlineRefusal{status: http.StatusBadGateway, msg: "could not read the instance"} |
| + body, err := json.Marshal(moves.OfflineActive{Host: d.host, Op: id, Account: d.acct.Sqid, App: d.app, At: time.Now().UTC()}) |
| + if err != nil { |
| + return unreadable |
| + } |
| + _, err = s.cfg.Releases.Put(ctx, moves.OfflineActiveKey, body, "") |
| + if err == nil { |
| + return nil |
| + } |
| + if !errors.Is(err, release.ErrCASConflict) { |
| + log.Printf("console: offline move %s: taking the lock: %v", d.host, err) |
| + return unreadable |
| + } |
| + raw, etag, err := s.cfg.Releases.Get(ctx, moves.OfflineActiveKey) |
| + var held moves.OfflineActive |
| + if err != nil || json.Unmarshal(raw, &held) != nil || held.Host == "" { |
| + log.Printf("console: offline move %s: reading the lock: %v", d.host, err) |
| + return unreadable |
| + } |
| + busy := &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s is moving offline. move one instance at a time", held.Host)} |
| + // A lock is taken before its record is written, so a record without the |
| + // move is only a stale lock once the holder has had time to write it. |
| + if time.Since(held.At) < offlineLockGrace { |
| + return busy |
| + } |
| + other, err := provision.GetRecord(ctx, s.cfg.Releases, held.Account, held.App, held.Host) |
| + switch { |
| + case errors.Is(err, os.ErrNotExist): |
| + case err != nil: |
| + log.Printf("console: offline move %s: reading the lock holder %s: %v", d.host, held.Host, err) |
| + return unreadable |
| + case other.Move.Offline() && other.Move.ID == held.Op: |
| + return busy |
| + } |
| + if _, err := s.cfg.Releases.Put(ctx, moves.OfflineActiveKey, body, etag); err != nil { |
| + if errors.Is(err, release.ErrCASConflict) { |
| + return busy |
| + } |
| + log.Printf("console: offline move %s: replacing a stale lock: %v", d.host, err) |
| + return unreadable |
| + } |
| + return nil |
| +} |
| + |
| +// offlineReleaseLock deletes the lock under the etag it read, when it is |
| +// this move's. A failure is logged and left: the next start finds the lock |
| +// stale (its record no longer carries the move) and replaces it. |
| +func (s *Server) offlineReleaseLock(ctx context.Context, host, id string) { |
| + ctx = context.WithoutCancel(ctx) |
| + raw, etag, err := s.cfg.Releases.Get(ctx, moves.OfflineActiveKey) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return |
| + } |
| + var held moves.OfflineActive |
| + if err != nil || json.Unmarshal(raw, &held) != nil { |
| + log.Printf("console: offline move %s: reading the lock to release it: %v", host, err) |
| + return |
| + } |
| + if held.Host != host || held.Op != id { |
| + return |
| + } |
| + if err := release.DeleteIfMatch(ctx, s.cfg.Releases, moves.OfflineActiveKey, etag); err != nil { |
| + log.Printf("console: offline move %s: releasing the lock: %v", host, err) |
| + } |
| } |
| // ---- the step each state waits for ---- |
| @@ -678,15 +791,22 @@ func (s *Server) offlineSealCheck(f offlineFleet, ev offlineEvidence, mv *provis |
| return sealer, boxes, nil |
| } |
| -// offlineFlipCheck: the seal's report is the one the record names and every |
| -// item it owns is OK; every source box has stopped and left no process or |
| -// layout journal, on a report this fresh. |
| +// offlineSealHolds says the record's source.evidence names the seal on disk, |
| +// byte for byte. The seal is written once, so the hash holds for the whole |
| +// move whatever the sealer's living report does each pass. |
| +func offlineSealHolds(ev offlineEvidence, mv *provision.MoveState, host string) bool { |
| + return ev.seal != nil && mv.Source != nil && |
| + offlineEvidenceMatches(mv.Source.Evidence, moves.OfflineSealKey(host, mv.ID), ev.seal.raw) |
| +} |
| + |
| +// offlineFlipCheck: the seal is the one the record names and every item the |
| +// source side reports is OK in it; every source box has stopped and left no |
| +// process or layout journal, on a report this fresh. |
| func (s *Server) offlineFlipCheck(ev offlineEvidence, mv *provision.MoveState, host, from string, now time.Time) *offlineRefusal { |
| - seal, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer) |
| - if !ok || mv.Source == nil || !offlineEvidenceMatches(mv.Source.Evidence, moves.OfflineReportKey(host, mv.ID, moves.OfflineRoleSealer, mv.Sealer), seal.raw) { |
| + if !offlineSealHolds(ev, mv, host) { |
| return offlineStateRefusal(mv.State, host) |
| } |
| - if items, _ := offlineSealItems(seal.OfflineReport); !items { |
| + if items, _ := offlineSealItems(ev.seal.OfflineReport); !items { |
| return offlineStateRefusal(mv.State, host) |
| } |
| for _, box := range mv.Boxes { |
| @@ -709,9 +829,9 @@ func processRefusal(box, host string) *offlineRefusal { |
| } |
| // offlineOpenCheck is the re-check of the stop: the target's landing report |
| -// has every landing item OK, the sealer's report is from this pass and holds |
| -// the tombstone for this move, and every source box is still held, fenced and |
| -// quiesced. |
| +// has every landing item OK, the seal is still the one the record names, the |
| +// sealer's living report is from this pass and holds the tombstone for this |
| +// move, and every source box is still held, fenced and quiesced. |
| func (s *Server) offlineOpenCheck(ev offlineEvidence, mv *provision.MoveState, host, from string, now time.Time) *offlineRefusal { |
| tgt := ev.byRole(moves.OfflineRoleTarget) |
| if len(tgt) != 1 { |
| @@ -720,6 +840,9 @@ func (s *Server) offlineOpenCheck(ev offlineEvidence, mv *provision.MoveState, h |
| if items, _ := offlineLandingItems(tgt[0].OfflineReport); !items { |
| return offlineStateRefusal(mv.State, host) |
| } |
| + if !offlineSealHolds(ev, mv, host) { |
| + return offlineStateRefusal(mv.State, host) |
| + } |
| seal, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer) |
| if !ok || !offlineFresh(seal.OfflineReport, now) || seal.TombstoneOwner != moves.TombstoneOwner(mv.ID) || !seal.TombstoneUntil.After(now) { |
| return offlineStateRefusal(mv.State, host) |
| @@ -757,13 +880,37 @@ func offlineFinishCheck(ev offlineEvidence, mv *provision.MoveState, host, targe |
| return refuse |
| } |
| for _, box := range boxes { |
| - if rep, ok := ev.get(moves.OfflineRoleSource, box); !ok || rep.Phase != sourcePhase { |
| + rep, ok := ev.get(moves.OfflineRoleSource, box) |
| + if !ok || rep.Phase != sourcePhase || !offlineRetiredTerminal(rep.Retired) { |
| return refuse |
| } |
| } |
| return nil |
| } |
| +// offlineRetiredTerminal says every path a parked report lists is settled: |
| +// parked, absent, or left with the reason it could not be parked (a |
| +// root-owned directory, a view mount). A left path is terminal; the move |
| +// goes on and the status view shows it. An unknown action, or a left with no |
| +// reason, is not a settled path. |
| +func offlineRetiredTerminal(retired []moves.RetiredPath) bool { |
| + for _, p := range retired { |
| + switch p.Action { |
| + case moves.RetiredParked, moves.RetiredAbsent: |
| + case moves.RetiredLeft: |
| + if p.Reason == "" { |
| + return false |
| + } |
| + default: |
| + return false |
| + } |
| + if p.Path == "" { |
| + return false |
| + } |
| + } |
| + return true |
| +} |
| + |
| // offlineVerbDoor is the shape of seal, flip, open and finish: the gate, an |
| // offline move in flight, the state the verb follows, the evidence, the |
| // verb's checks, one transition. |
| @@ -827,6 +974,14 @@ type offlineReportView struct { |
| Fresh bool `json:"fresh"` |
| } |
| +// offlineLeftView is a path a source box could not park and left in place, |
| +// with the reason it gave (the parked report's inventory). |
| +type offlineLeftView struct { |
| + Box string `json:"box"` |
| + Path string `json:"path"` |
| + Reason string `json:"reason"` |
| +} |
| + |
| // offlineItemView is one row of the manifest with the latest check of it. |
| // Status is ok, waiting, failed or unchecked (no report decides the item). |
| type offlineItemView struct { |
| @@ -843,7 +998,8 @@ type offlineItemView struct { |
| // State is "none" when no offline move is in flight (Last then carries the |
| // outcome of the one that ended). Next is the verb that moves the move on |
| // right now; empty while a box has work to do. Waiting is the sentence that |
| -// verb would be refused with. CanAbort says abort is legal. |
| +// verb would be refused with. CanAbort says abort is legal. Failed says the |
| +// sealer or the target reported it could not do its step: abort is the way on. |
| type offlineView struct { |
| Host string `json:"host"` |
| State string `json:"state"` |
| @@ -859,6 +1015,8 @@ type offlineView struct { |
| Next string `json:"next,omitempty"` |
| Waiting string `json:"waiting,omitempty"` |
| CanAbort bool `json:"can_abort,omitempty"` |
| + Failed bool `json:"failed,omitempty"` |
| + Left []offlineLeftView `json:"left,omitempty"` |
| Reports []offlineReportView `json:"reports,omitempty"` |
| Items []offlineItemView `json:"items,omitempty"` |
| Last *provision.MoveOutcome `json:"last,omitempty"` |
| @@ -886,7 +1044,7 @@ func (s *Server) offlineViewOf(ctx context.Context, rec *provision.Record) offli |
| } |
| ev, err := s.readOfflineEvidence(ctx, rec.Host, mv.ID) |
| if err != nil { |
| - log.Printf("console: offline move %s: reports: %v", rec.Host, err) |
| + log.Printf("console: offline move %s: reading the reports: %v", rec.Host, err) |
| v.Waiting = "could not read the move's reports. try again" |
| return v |
| } |
| @@ -894,6 +1052,13 @@ func (s *Server) offlineViewOf(ctx context.Context, rec *provision.Record) offli |
| v.Reports = append(v.Reports, offlineReportView{OfflineReport: r.OfflineReport, Fresh: offlineFresh(r.OfflineReport, now)}) |
| } |
| v.Items = offlineItems(ev, mv) |
| + for _, r := range ev.byRole(moves.OfflineRoleSource) { |
| + for _, p := range r.Retired { |
| + if p.Action == moves.RetiredLeft { |
| + v.Left = append(v.Left, offlineLeftView{Box: r.Box, Path: p.Path, Reason: p.Reason}) |
| + } |
| + } |
| + } |
| if verb := offlineVerbOf(mv.State); verb != "" { |
| d := offlineDoor{host: rec.Host} |
| if _, ref := s.offlineStepFor(ctx, d, mv, ev, now); ref != nil { |
| @@ -902,11 +1067,12 @@ func (s *Server) offlineViewOf(ctx context.Context, rec *provision.Record) offli |
| v.Next = verb |
| } |
| } |
| - // Landing waits for a box, unless the box says it cannot land. |
| - if mv.State == provision.OfflineLanding { |
| - for _, r := range ev.byRole(moves.OfflineRoleTarget) { |
| - if r.Phase == moves.OfflinePhaseFailed { |
| - v.Waiting = fmt.Sprintf("%s could not land %s. run carlos move offline abort --host %s", r.Box, rec.Host, rec.Host) |
| + // A box that says it failed (the sealer, or the target that cannot |
| + // land) means abort is the way on. The CLI says it with MV-O6. |
| + if v.CanAbort { |
| + for _, r := range ev.reports { |
| + if r.Phase == moves.OfflinePhaseFailed && (r.Role == moves.OfflineRoleSealer || r.Role == moves.OfflineRoleTarget) { |
| + v.Failed = true |
| } |
| } |
| } |
| @@ -924,6 +1090,11 @@ func offlineItems(ev offlineEvidence, mv *provision.MoveState) []offlineItemView |
| if r, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer); ok { |
| owners[moves.OfflineOwnerSeal] = []offlineReport{r} |
| } |
| + if ev.seal != nil { |
| + // The seal is what the sealer decided; the living report is the |
| + // latest word on the same items, and is asked first. |
| + owners[moves.OfflineOwnerSeal] = append(owners[moves.OfflineOwnerSeal], *ev.seal) |
| + } |
| out := make([]offlineItemView, 0, len(moves.OfflineManifest)) |
| for _, it := range moves.OfflineManifest { |
| row := offlineItemView{ID: it.ID, Item: it.Item, Disposition: it.Disposition, Owner: it.Owner, Source: it.Source, Status: "waiting"} |
| @@ -960,18 +1131,3 @@ func (s *Server) handleAPIInstanceOfflineStatus(w http.ResponseWriter, r *http.R |
| } |
| writeJSON(w, http.StatusOK, s.offlineViewOf(r.Context(), d.rec)) |
| } |
| - |
| -// registerOfflineMoveRoutes registers the offline move's doors. They are |
| -// always registered: a console on the claim path or with Phase 1 on answers |
| -// every one with MV-O20, after the operator gate. |
| -func (s *Server) registerOfflineMoveRoutes() { |
| - const base = "/api/cli/apps/{account}/{app}/instances/{host}/move/offline" |
| - inst := svcrole.Allow(svcrole.Instance) |
| - s.cliRoute("GET "+base, inst, s.handleAPIInstanceOfflineStatus) |
| - s.cliRoute("POST "+base, inst, s.handleAPIInstanceOfflineStart) |
| - s.cliRoute("POST "+base+"/seal", inst, s.offlineVerbDoor(offlineVerbSeal)) |
| - s.cliRoute("POST "+base+"/flip", inst, s.offlineVerbDoor(offlineVerbFlip)) |
| - s.cliRoute("POST "+base+"/open", inst, s.offlineVerbDoor(offlineVerbOpen)) |
| - s.cliRoute("POST "+base+"/finish", inst, s.offlineVerbDoor(offlineVerbFinish)) |
| - s.cliRoute("POST "+base+"/abort", inst, s.handleAPIInstanceOfflineAbort) |
| -} |
diff --git a/internal/console/apiinstancemove_offline_test.go b/internal/console/apiinstancemove_offline_test.go| index 49a7776e8..7e453142f 100644 |
| --- a/internal/console/apiinstancemove_offline_test.go |
| +++ b/internal/console/apiinstancemove_offline_test.go |
| @@ -8,6 +8,7 @@ import ( |
| "fmt" |
| "net/http" |
| "strings" |
| + "sync" |
| "testing" |
| "time" |
| @@ -305,19 +306,38 @@ func sealerChecks() map[string]moves.Check { |
| return out |
| } |
| -// sealed plays the sealer: its report, and the transition that records |
| -// where it is. |
| +// putSeal writes seal.json as the sealer does: once and create-only when it |
| +// is not there, and replaced only by a test that tampers with it. |
| +func (w *offlineWorld) putSeal(sealer string, edit func(*moves.OfflineReport)) []byte { |
| + w.t.Helper() |
| + r := moves.OfflineReport{V: moves.OfflineReportVersion, Op: w.id, Host: offlineHost, Box: sealer, Role: moves.OfflineRoleSealer, |
| + Phase: moves.OfflinePhaseSealed, At: time.Now().UTC(), Items: sealerChecks(), FinalTXID: 7, TombstoneOwner: moves.TombstoneOwner(w.id)} |
| + if edit != nil { |
| + edit(&r) |
| + } |
| + body, err := json.Marshal(r) |
| + if err != nil { |
| + w.t.Fatal(err) |
| + } |
| + key := moves.OfflineSealKey(offlineHost, w.id) |
| + if _, _, err := w.fx.rel.Get(context.Background(), key); err != nil { |
| + if _, err := w.fx.rel.Put(context.Background(), key, body, ""); err != nil { |
| + w.t.Fatal(err) |
| + } |
| + } else { |
| + offlinePut(w.t, w.fx.rel, key, body) |
| + } |
| + return body |
| +} |
| + |
| +// sealed plays the sealer: the seal, its living report, and the transition |
| +// that records where the seal is. It returns the seal's bytes. |
| func (w *offlineWorld) sealed(sealer string) []byte { |
| w.t.Helper() |
| - raw := w.report(moves.OfflineRoleSealer, sealer, moves.OfflinePhaseSealed, func(r *moves.OfflineReport) { |
| - r.Items = sealerChecks() |
| - r.TombstoneOwner = moves.TombstoneOwner(w.id) |
| - r.TombstoneUntil = time.Now().Add(moves.TombstoneTTL) |
| - }) |
| - sum := sha256.Sum256(raw) |
| - key := moves.OfflineReportKey(offlineHost, w.id, moves.OfflineRoleSealer, sealer) |
| - w.edit(provision.OfflineSealedBy(w.id, sealer, provision.MoveSource{FinalTXID: 7, Evidence: key + "@" + hex.EncodeToString(sum[:])})) |
| - return raw |
| + seal := w.putSeal(sealer, nil) |
| + w.sealerFresh(sealer) |
| + w.edit(provision.OfflineSealedBy(w.id, sealer, provision.MoveSource{FinalTXID: 7, Evidence: moves.OfflineSealEvidence(offlineHost, w.id, seal)})) |
| + return seal |
| } |
| // landed plays the target: its landing report and the transition. |
| @@ -337,6 +357,9 @@ func (w *offlineWorld) toLanded() { |
| w.stopped(nil, "ie-box") |
| w.ok("/seal", "", provision.OfflineSealing) |
| w.sealed("ie-box") |
| + // The sealer rewrites its living report on every pass, so by the time |
| + // the operator flips it is not the bytes it was at the seal. The seal is. |
| + w.sealerFresh("ie-box") |
| w.stopped(nil, "ie-box") |
| w.ok("/flip", "", provision.OfflineLanding) |
| w.landed("uk-box") |
| @@ -462,14 +485,11 @@ func TestOfflineStartRefusals(t *testing.T) { |
| } |
| const body = `{"region":"uk"}` |
| other := func(w *offlineWorld) { |
| - // A second instance, moving offline. |
| - u := instancesURL(w.fx, "bab", "titogo") |
| - if c, _, out := rawReq(t, http.MethodPost, u, w.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated { |
| - t.Fatalf("create = %d %s", c, out) |
| - } |
| - if _, err := provision.SetRecordMove(context.Background(), w.fx.rel, "bab", "titogo", "beta.go.tito.io", |
| - provision.OfflineStart("mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", "ie", "uk", "x", time.Now())); err != nil { |
| - t.Fatal(err) |
| + // A second instance, moving offline: started through its own door, so |
| + // it holds the lock. |
| + v, code, out := w.startOther("beta.go.tito.io") |
| + if code != http.StatusAccepted { |
| + t.Fatalf("beta start = %d %s (%+v)", code, out, v) |
| } |
| } |
| cases := []tc{ |
| @@ -725,23 +745,47 @@ func TestOfflineFlipDoor(t *testing.T) { |
| w.stopped(nil, "ie-box") |
| w.ok("/seal", "", provision.OfflineSealing) |
| w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealing)) |
| - // The sealer's report is not what the record names. |
| - w.sealed("ie-box") |
| + seal := w.sealed("ie-box") |
| w.stopped(nil, "ie-box") |
| + sealKey := moves.OfflineSealKey(offlineHost, w.id) |
| + // seal.json is create-only: a second write does not replace it. |
| + if _, err := w.fx.rel.Put(context.Background(), sealKey, []byte("{}"), ""); err == nil { |
| + t.Fatal("the seal was written twice") |
| + } |
| + // The sealer's living report is rewritten on every pass and the seal is |
| + // not: nothing the living report does moves the hash flip checks. |
| + w.sealerFresh("ie-box") |
| w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) { |
| - r.Items = sealerChecks() |
| - r.FinalTXID = 99 // a different report from the one the record hashed |
| + r.Items = map[string]moves.Check{} // the living report carries no items at all now |
| + r.TombstoneOwner = moves.TombstoneOwner(w.id) |
| + r.TombstoneUntil = time.Now().Add(time.Hour) |
| + r.FinalTXID = 99 |
| }) |
| + // A seal that is not the one the record names. |
| + tampered := w.putSeal("ie-box", func(r *moves.OfflineReport) { r.FinalTXID = 99 }) |
| + if string(tampered) == string(seal) { |
| + t.Fatal("the tampered seal is the seal") |
| + } |
| w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed)) |
| - // A seal item that is not OK. |
| - raw := w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) { |
| - r.Items = sealerChecks() |
| + offlinePut(t, w.fx.rel, sealKey, seal) |
| + // No seal at all. |
| + if err := w.fx.rel.Delete(context.Background(), sealKey); err != nil { |
| + t.Fatal(err) |
| + } |
| + w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed)) |
| + offlinePut(t, w.fx.rel, sealKey, seal) |
| + // A seal item that is not OK, and a seal of another move's tombstone, each |
| + // with a hash the record names. |
| + bad := w.putSeal("ie-box", func(r *moves.OfflineReport) { |
| r.Items["files"] = moves.Check{OK: false, Reason: "a file the archive refuses"} |
| }) |
| - w.rehash("ie-box", raw) |
| + w.rehash(bad) |
| + w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed)) |
| + bad = w.putSeal("ie-box", func(r *moves.OfflineReport) { r.TombstoneOwner = moves.TombstoneOwner("mv_other") }) |
| + w.rehash(bad) |
| w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed)) |
| - good := w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) { r.Items = sealerChecks() }) |
| - w.rehash("ie-box", good) |
| + offlinePut(t, w.fx.rel, sealKey, seal) |
| + w.rehash(seal) |
| // A process left, a layout journal, a stale report. |
| w.stopped(func(_ string, r *moves.OfflineReport) { r.Procs = 1 }, "ie-box") |
| w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box")) |
| @@ -766,14 +810,12 @@ func TestOfflineFlipDoor(t *testing.T) { |
| } |
| } |
| -// rehash points the record's source evidence at a report's bytes, as the |
| -// sealer's transition does when it records its own report. |
| -func (w *offlineWorld) rehash(sealer string, raw []byte) { |
| +// rehash points the record's source evidence at a seal's bytes, as the |
| +// sealer's transition records its own seal. |
| +func (w *offlineWorld) rehash(seal []byte) { |
| w.t.Helper() |
| - sum := sha256.Sum256(raw) |
| - key := moves.OfflineReportKey(offlineHost, w.id, moves.OfflineRoleSealer, sealer) |
| w.edit(func(r *provision.Record) error { |
| - r.Move.Source.Evidence = key + "@" + hex.EncodeToString(sum[:]) |
| + r.Move.Source.Evidence = moves.OfflineSealEvidence(offlineHost, w.id, seal) |
| return nil |
| }) |
| } |
| @@ -822,6 +864,15 @@ func TestOfflineOpenDoorRechecksTheStop(t *testing.T) { |
| w.stopped(func(_ string, r *moves.OfflineReport) { r.At = time.Now().Add(-10 * time.Minute) }, "ie-box") |
| w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgMC10("ie-box")) |
| reset() |
| + // The seal changed under the move. |
| + sealKey := moves.OfflineSealKey(offlineHost, w.id) |
| + sealRaw, _, err := w.fx.rel.Get(context.Background(), sealKey) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + offlinePut(t, w.fx.rel, sealKey, append(append([]byte{}, sealRaw...), ' ')) |
| + w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO25(provision.OfflineLanded)) |
| + offlinePut(t, w.fx.rel, sealKey, sealRaw) |
| // A landing item that is not OK. |
| w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseLanded, func(r *moves.OfflineReport) { |
| r.Items = okChecks(moves.OfflineOwnerLanding) |
| @@ -860,7 +911,30 @@ func TestOfflineFinishDoor(t *testing.T) { |
| w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineOpen)) |
| w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseOpened, func(r *moves.OfflineReport) { r.Items = okChecks(moves.OfflineOwnerLanding) }) |
| w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineOpen)) |
| - w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, nil) |
| + // A parked report whose inventory is not settled is not parked. |
| + for name, retired := range map[string][]moves.RetiredPath{ |
| + "an unknown action": {{Path: "/data/acme.go.tito.io", Action: "deleted"}}, |
| + "left, no reason": {{Path: "/opt/carlos/live/acme.go.tito.io", Action: moves.RetiredLeft}}, |
| + "no path": {{Action: moves.RetiredParked}}, |
| + } { |
| + w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, func(r *moves.OfflineReport) { r.Retired = retired }) |
| + code, out := w.req(http.MethodPost, "/finish", "") |
| + if code != http.StatusConflict || errorText(out) != msgO25(provision.OfflineOpen) { |
| + t.Fatalf("%s: %d %s", name, code, out) |
| + } |
| + } |
| + // A path the box could not park is terminal when it says why, and the |
| + // status view shows it. |
| + w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, func(r *moves.OfflineReport) { |
| + r.Retired = []moves.RetiredPath{ |
| + {Path: "/data/acme.go.tito.io", Action: moves.RetiredParked}, |
| + {Path: "/data/acme.go.tito.io.env", Action: moves.RetiredAbsent}, |
| + {Path: "/opt/carlos/live/acme.go.tito.io", Action: moves.RetiredLeft, Reason: "root-owned"}, |
| + } |
| + }) |
| + if v := w.status(); v.Next != offlineVerbFinish || len(v.Left) != 1 || v.Left[0] != (offlineLeftView{Box: "ie-box", Path: "/opt/carlos/live/acme.go.tito.io", Reason: "root-owned"}) { |
| + t.Fatalf("status = next %q left %+v waiting %q", v.Next, v.Left, v.Waiting) |
| + } |
| before := serialBytes(t, w.fx.rel) |
| w.ok("/finish", "", provision.OfflineFinishing) |
| if serialBytes(t, w.fx.rel) == before { |
| @@ -959,6 +1033,9 @@ func TestOfflineStatusView(t *testing.T) { |
| if v.State != provision.OfflineLanded || v.ID != w.id || v.Sealer != "ie-box" || v.From != "ie" || v.To != "uk" || !v.CanAbort || v.Next != offlineVerbOpen { |
| t.Fatalf("status = %+v", v) |
| } |
| + if v.Failed { |
| + t.Error("a healthy move is marked failed") |
| + } |
| roles := map[string]bool{} |
| for _, r := range v.Reports { |
| roles[r.Role+"/"+r.Box] = true |
| @@ -992,6 +1069,9 @@ func TestOfflineStatusView(t *testing.T) { |
| delete(r.Items, "release") |
| r.Items["files"] = moves.Check{OK: false, Reason: "the digest differs"} |
| }) |
| + if !w.status().Failed { |
| + t.Error("a sealer that reported failed is not marked failed in the view") |
| + } |
| seen := map[string]offlineItemView{} |
| for _, it := range w.status().Items { |
| seen[it.ID] = it |
| @@ -1125,4 +1205,181 @@ func TestOfflineMovesOutOfAndIntoHome(t *testing.T) { |
| named := offlineBox{Box: "ie-named", Placement: "ie", Bucket: "data-ie"} |
| c := newOfflineWorldIn(t, "ie", "uk", homeBox, ukOff("uk-box"), named) |
| c.refused(http.MethodPost, "", `{"region":"ie"}`, http.StatusConflict, "ie-named sets CARLOS_REGION, so it cannot take a move into the home region") |
| + |
| + // And out of home, where its published placement is not the record's. |
| + d := newOfflineWorldIn(t, "ie", "ie", homeBox, ukOff("uk-box"), named) |
| + d.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "ie-named sets CARLOS_REGION, so it cannot take a move into the home region") |
| +} |
| + |
| +// startOther creates a second instance and starts it through its own door. |
| +func (w *offlineWorld) startOther(host string) (offlineView, int, string) { |
| + w.t.Helper() |
| + u := instancesURL(w.fx, "bab", "titogo") |
| + if c, _, out := rawReq(w.t, http.MethodPost, u, w.token, `{"host":"`+host+`","region":"ie"}`); c != http.StatusCreated && c != http.StatusConflict { |
| + w.t.Fatalf("create %s = %d %s", host, c, out) |
| + } |
| + code, _, out := rawReq(w.t, http.MethodPost, u+"/"+host+"/move/offline", w.token, `{"region":"uk"}`) |
| + var v offlineView |
| + _ = json.Unmarshal([]byte(out), &v) |
| + return v, code, out |
| +} |
| + |
| +func (w *offlineWorld) lock() (moves.OfflineActive, bool) { |
| + w.t.Helper() |
| + raw, _, err := w.fx.rel.Get(context.Background(), moves.OfflineActiveKey) |
| + if err != nil { |
| + return moves.OfflineActive{}, false |
| + } |
| + var a moves.OfflineActive |
| + if err := json.Unmarshal(raw, &a); err != nil { |
| + w.t.Fatal(err) |
| + } |
| + return a, true |
| +} |
| + |
| +// Two starts at once for different hosts: exactly one wins the lock. |
| +func TestOfflineConcurrentStartsOneWins(t *testing.T) { |
| + w := newOfflineWorld(t, oneEach...) |
| + u := instancesURL(w.fx, "bab", "titogo") |
| + if c, _, out := rawReq(t, http.MethodPost, u, w.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated { |
| + t.Fatalf("create = %d %s", c, out) |
| + } |
| + hosts := []string{offlineHost, "beta.go.tito.io"} |
| + type answer struct { |
| + code int |
| + out string |
| + } |
| + got := make([]answer, len(hosts)) |
| + var wg sync.WaitGroup |
| + gate := make(chan struct{}) |
| + for i, h := range hosts { |
| + wg.Add(1) |
| + go func() { |
| + defer wg.Done() |
| + <-gate |
| + c, _, out := rawReq(t, http.MethodPost, u+"/"+h+"/move/offline", w.token, `{"region":"uk"}`) |
| + got[i] = answer{c, out} |
| + }() |
| + } |
| + close(gate) |
| + wg.Wait() |
| + won, lost := -1, -1 |
| + for i, a := range got { |
| + switch a.code { |
| + case http.StatusAccepted: |
| + won = i |
| + case http.StatusConflict: |
| + lost = i |
| + default: |
| + t.Fatalf("start %s = %d %s", hosts[i], a.code, a.out) |
| + } |
| + } |
| + if won < 0 || lost < 0 { |
| + t.Fatalf("want one winner and one loser: %+v", got) |
| + } |
| + if want := hosts[won] + " is moving offline. move one instance at a time"; errorText(got[lost].out) != want { |
| + t.Fatalf("the loser was told %q, want %q", errorText(got[lost].out), want) |
| + } |
| + if a, ok := w.lock(); !ok || a.Host != hosts[won] { |
| + t.Fatalf("lock = %+v %v, want the winner %s", a, ok, hosts[won]) |
| + } |
| + loser, err := provision.GetRecord(context.Background(), w.fx.rel, "bab", "titogo", hosts[lost]) |
| + if err != nil || loser.Move != nil { |
| + t.Fatalf("the loser's record = %+v %v", loser.Move, err) |
| + } |
| +} |
| + |
| +// The lock is released when the move ends, by finish and by the finish that |
| +// ends an abort, and a stale lock does not stop the next start. |
| +func TestOfflineLockLifecycle(t *testing.T) { |
| + w := newOfflineWorld(t, oneEach...) |
| + w.toLanded() |
| + if a, ok := w.lock(); !ok || a.Host != offlineHost || a.Op != w.id || a.Account != "bab" || a.App != "titogo" { |
| + t.Fatalf("lock = %+v %v", a, ok) |
| + } |
| + w.ok("/open", "", provision.OfflineOpen) |
| + w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseOpened, func(r *moves.OfflineReport) { r.Items = okChecks(moves.OfflineOwnerLanding) }) |
| + w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, nil) |
| + w.ok("/finish", "", provision.OfflineFinishing) |
| + if _, ok := w.lock(); !ok { |
| + t.Fatal("the lock was released before the move ended") |
| + } |
| + w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseLifted, nil) |
| + w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseGone, nil) |
| + w.ok("/finish", "", "none") |
| + if _, ok := w.lock(); ok { |
| + t.Fatal("finish left the lock") |
| + } |
| + |
| + // An abort ends at finish too. |
| + b := newOfflineWorld(t, oneEach...) |
| + b.start() |
| + b.ok("/abort", "", provision.OfflineReverting) |
| + if _, ok := b.lock(); !ok { |
| + t.Fatal("abort released the lock before the finish") |
| + } |
| + b.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil) |
| + b.ok("/finish", "", "none") |
| + if _, ok := b.lock(); ok { |
| + t.Fatal("the finish after an abort left the lock") |
| + } |
| + // Another start now takes it. |
| + b.start() |
| + if a, ok := b.lock(); !ok || a.Op != b.id { |
| + t.Fatalf("lock = %+v %v", a, ok) |
| + } |
| +} |
| + |
| +func TestOfflineStaleLockIsReplaced(t *testing.T) { |
| + put := func(w *offlineWorld, a moves.OfflineActive) { |
| + body, _ := json.Marshal(a) |
| + offlinePut(t, w.fx.rel, moves.OfflineActiveKey, body) |
| + } |
| + // A lock naming a host with no record. |
| + w := newOfflineWorld(t, oneEach...) |
| + put(w, moves.OfflineActive{Host: "gone.go.tito.io", Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "titogo", At: time.Now().Add(-time.Hour)}) |
| + w.start() |
| + if a, _ := w.lock(); a.Host != offlineHost || a.Op != w.id { |
| + t.Fatalf("lock = %+v", a) |
| + } |
| + // A lock naming a host whose record carries no such move (a failed release). |
| + x := newOfflineWorld(t, oneEach...) |
| + if c, _, out := rawReq(t, http.MethodPost, instancesURL(x.fx, "bab", "titogo"), x.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated { |
| + t.Fatalf("create = %d %s", c, out) |
| + } |
| + put(x, moves.OfflineActive{Host: "beta.go.tito.io", Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "titogo", At: time.Now().Add(-time.Hour)}) |
| + x.start() |
| + // A lock whose record carries another move of that host is not this one's. |
| + y := newOfflineWorld(t, oneEach...) |
| + put(y, moves.OfflineActive{Host: offlineHost, Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "titogo", At: time.Now().Add(-time.Hour)}) |
| + y.start() |
| + // A young lock is believed even before its record carries the move: the |
| + // start door takes the lock, then writes the record. |
| + f := newOfflineWorld(t, oneEach...) |
| + if c, _, out := rawReq(t, http.MethodPost, instancesURL(f.fx, "bab", "titogo"), f.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated { |
| + t.Fatalf("create = %d %s", c, out) |
| + } |
| + put(f, moves.OfflineActive{Host: "beta.go.tito.io", Op: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Account: "bab", App: "titogo", At: time.Now()}) |
| + f.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "beta.go.tito.io is moving offline. move one instance at a time") |
| + // An old lock is live while its record carries the move. |
| + g := newOfflineWorld(t, oneEach...) |
| + if _, code, out := g.startOther("beta.go.tito.io"); code != http.StatusAccepted { |
| + t.Fatalf("beta start = %d %s", code, out) |
| + } |
| + a, _ := g.lock() |
| + a.At = time.Now().Add(-24 * time.Hour) |
| + put(g, a) |
| + g.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "beta.go.tito.io is moving offline. move one instance at a time") |
| + // An unreadable lock is not guessed at. |
| + z := newOfflineWorld(t, oneEach...) |
| + offlinePut(t, z.fx.rel, moves.OfflineActiveKey, []byte("not json")) |
| + z.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusBadGateway, "could not read the instance") |
| +} |
| + |
| +// A record nobody can read elsewhere does not block a start: the lock is the |
| +// gate, not a listing of every record. |
| +func TestOfflineStartIgnoresOtherRecords(t *testing.T) { |
| + w := newOfflineWorld(t, oneEach...) |
| + offlinePut(t, w.fx.rel, provision.RecordKey("bab", "titogo", "junk.go.tito.io"), []byte("{not a record")) |
| + w.start() |
| } |
diff --git a/internal/console/authcli.go b/internal/console/authcli.go| index 8733e88f6..efcdb15d7 100644 |
| --- a/internal/console/authcli.go |
| +++ b/internal/console/authcli.go |
| @@ -718,7 +718,13 @@ func (s *Server) registerCLIRoutes() { |
| } |
| // The offline operator move's doors: always registered, operator-only, |
| // and refused with MV-O20 on a console with host claims or Phase 1 on. |
| - s.registerOfflineMoveRoutes() |
| + s.cliRoute("GET /api/cli/apps/{account}/{app}/instances/{host}/move/offline", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceOfflineStatus) |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceOfflineStart) |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/seal", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbSeal)) |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/flip", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbFlip)) |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/open", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbOpen)) |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/finish", svcrole.Allow(svcrole.Instance), s.offlineVerbDoor(offlineVerbFinish)) |
| + s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/abort", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceOfflineAbort) |
| s.cliRoute("DELETE /api/cli/apps/{account}/{app}/instances/{host}", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceDelete) |
| // The import door (seeded-instances spec §5): the instance role, like |
| // the create door that begins it. |
diff --git a/internal/console/svcroutes_test.go b/internal/console/svcroutes_test.go| index 1777ac98b..f46127714 100644 |
| --- a/internal/console/svcroutes_test.go |
| +++ b/internal/console/svcroutes_test.go |
| @@ -529,11 +529,21 @@ var exemptFromLegacyPin = map[string]bool{ |
| "POST /api/cli/apps/{account}/{app}/instances/{host}/move": true, |
| "POST /api/cli/apps/{account}/{app}/instances/{host}/move/cut": true, |
| "DELETE /api/cli/apps/{account}/{app}/instances/{host}/move": true, |
| - "GET /api/cli/apps/{account}/{app}/sidecars": true, |
| - "POST /api/cli/apps/{account}/{app}/sidecars": true, |
| - "GET /api/cli/apps/{account}/{app}/sidecars/{name}": true, |
| - "DELETE /api/cli/apps/{account}/{app}/sidecars/{name}": true, |
| - "POST /api/cli/apps/{account}/{app}/sidecars/{name}/wake": true, |
| + // The offline operator move's doors: each is operator-only behind |
| + // moveOperatorGate (a service token meets the family's 404), and |
| + // refuses with MV-O20 on a console with host claims or Phase 1 on. |
| + "GET /api/cli/apps/{account}/{app}/instances/{host}/move/offline": true, |
| + "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline": true, |
| + "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/seal": true, |
| + "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/flip": true, |
| + "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/open": true, |
| + "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/finish": true, |
| + "POST /api/cli/apps/{account}/{app}/instances/{host}/move/offline/abort": true, |
| + "GET /api/cli/apps/{account}/{app}/sidecars": true, |
| + "POST /api/cli/apps/{account}/{app}/sidecars": true, |
| + "GET /api/cli/apps/{account}/{app}/sidecars/{name}": true, |
| + "DELETE /api/cli/apps/{account}/{app}/sidecars/{name}": true, |
| + "POST /api/cli/apps/{account}/{app}/sidecars/{name}/wake": true, |
| // The platform's GeoIP lookup (geoip spec §11.1) is new access on the |
| // analytics read's footing: it did not exist before service principals, |
| // and the app itself is the consumer (a batch job or a webhook handler |
diff --git a/internal/moves/offline.go b/internal/moves/offline.go| index a51bdbbe1..b9da2fc97 100644 |
| --- a/internal/moves/offline.go |
| +++ b/internal/moves/offline.go |
| @@ -1,6 +1,8 @@ |
| package moves |
| import ( |
| + "crypto/sha256" |
| + "encoding/hex" |
| "fmt" |
| "path/filepath" |
| "time" |
| @@ -323,6 +325,68 @@ func OfflineReportPrefix(host, op string) string { |
| return "control/offline-moves/" + host + "/" + op + "/" |
| } |
| +// OfflineSealKey is where the sealer writes the seal, once and create-only: |
| +// control/offline-moves/<host>/<op>/seal.json. Unlike a report it is never |
| +// rewritten. It is an OfflineReport (role sealer, phase sealed) holding what |
| +// the sealer decided and recorded at the seal: the sealed position |
| +// (FinalTXID, Staged), the files and env digests, the env pin, the release, |
| +// the timers digest, the replication rung and next due, TombstoneOwner |
| +// (moved:<op>), and Items, an OK Check for every manifest item with Source |
| +// "seal". At is when it was sealed; TombstoneUntil is zero (the live |
| +// expiry is the sealer's per-pass report). The record's source.evidence is |
| +// this key and the sha256 of its bytes (OfflineSealEvidence). "" for a host |
| +// that fails provision.ValidHostShape or an unusable op. |
| +func OfflineSealKey(host, op string) string { |
| + if !provision.ValidHostShape(host) || !offlineSegment(op) { |
| + return "" |
| + } |
| + return "control/offline-moves/" + host + "/" + op + "/seal.json" |
| +} |
| + |
| +// OfflineSealEvidence is source.evidence for a seal written as raw: the seal |
| +// key, "@", and the hex sha256 of the bytes. "" for a bad host or op. |
| +func OfflineSealEvidence(host, op string, raw []byte) string { |
| + key := OfflineSealKey(host, op) |
| + if key == "" { |
| + return "" |
| + } |
| + sum := sha256.Sum256(raw) |
| + return key + "@" + hex.EncodeToString(sum[:]) |
| +} |
| + |
| +// ValidateOfflineSeal reports why a seal cannot be trusted as the seal of op |
| +// on host: it is a valid sealer report of that move, phase sealed, that names |
| +// the move's tombstone owner and carries an OK Check for every item the |
| +// source side reports. |
| +func ValidateOfflineSeal(r OfflineReport, host, op string) error { |
| + if err := ValidateOfflineReport(r); err != nil { |
| + return err |
| + } |
| + if r.Host != host || r.Op != op || r.Role != OfflineRoleSealer || r.Phase != OfflinePhaseSealed { |
| + return fmt.Errorf("offline seal: not the sealed report of this move") |
| + } |
| + if r.TombstoneOwner != TombstoneOwner(op) { |
| + return fmt.Errorf("offline seal: tombstone owner %q, want %q", r.TombstoneOwner, TombstoneOwner(op)) |
| + } |
| + if ok, bad := r.SourceItemsOK(); !ok { |
| + return fmt.Errorf("offline seal: items not OK: %v", bad) |
| + } |
| + return nil |
| +} |
| + |
| +// OfflineActiveKey is the one lock of the offline move: while it names a |
| +// host whose record still carries that move, no other offline move starts. |
| +const OfflineActiveKey = "control/offline-moves/active.json" |
| + |
| +// OfflineActive is the lock's body. |
| +type OfflineActive struct { |
| + Host string `json:"host"` |
| + Op string `json:"op"` |
| + Account string `json:"account"` |
| + App string `json:"app"` |
| + At time.Time `json:"at"` |
| +} |
| + |
| // OfflineArchivePrefix is where the move's sealed files archive and its |
| // timers snapshot live in the target bucket: moves/@offline/<host>/<op>/. The |
| // "@" cannot start a host, so it never collides with a host's own staging |
diff --git a/internal/moves/offline_test.go b/internal/moves/offline_test.go| index 635eb9e23..84a0e4050 100644 |
| --- a/internal/moves/offline_test.go |
| +++ b/internal/moves/offline_test.go |
| @@ -441,3 +441,55 @@ func TestOfflineStateAndPhaseNamesAreDistinct(t *testing.T) { |
| t.Fatalf("kind %q", provision.MoveKindOffline) |
| } |
| } |
| + |
| +func TestOfflineSealKeyAndValidation(t *testing.T) { |
| + const host, op = "acme.go.tito.io", "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa" |
| + if got := OfflineSealKey(host, op); got != "control/offline-moves/"+host+"/"+op+"/seal.json" { |
| + t.Fatalf("seal key = %q", got) |
| + } |
| + for _, bad := range [][2]string{{"", op}, {"not a host", op}, {host, ""}, {host, "../x"}, {host, "a/b"}} { |
| + if OfflineSealKey(bad[0], bad[1]) != "" || OfflineSealEvidence(bad[0], bad[1], []byte("x")) != "" { |
| + t.Errorf("key for %q accepted", bad) |
| + } |
| + } |
| + // The seal key is not a report key and sits under the report prefix. |
| + if !strings.HasPrefix(OfflineSealKey(host, op), OfflineReportPrefix(host, op)) { |
| + t.Error("the seal is outside the move's prefix") |
| + } |
| + items := map[string]Check{} |
| + for _, it := range OfflineSourced() { |
| + items[it.ID] = Check{OK: true} |
| + } |
| + good := OfflineReport{V: OfflineReportVersion, Op: op, Host: host, Box: "ie-box", Role: OfflineRoleSealer, Phase: OfflinePhaseSealed, |
| + Items: items, TombstoneOwner: TombstoneOwner(op), At: time.Now()} |
| + if err := ValidateOfflineSeal(good, host, op); err != nil { |
| + t.Fatal(err) |
| + } |
| + for name, edit := range map[string]func(*OfflineReport){ |
| + "another host": func(r *OfflineReport) { r.Host = "other.go.tito.io" }, |
| + "another op": func(r *OfflineReport) { r.Op = "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb" }, |
| + "a source role": func(r *OfflineReport) { r.Role = OfflineRoleSource }, |
| + "not sealed": func(r *OfflineReport) { r.Phase = OfflinePhaseStopped }, |
| + "no tombstone": func(r *OfflineReport) { r.TombstoneOwner = "" }, |
| + "an item not OK": func(r *OfflineReport) { |
| + c := map[string]Check{} |
| + for k, v := range items { |
| + c[k] = v |
| + } |
| + c["files"] = Check{} |
| + r.Items = c |
| + }, |
| + "a missing item": func(r *OfflineReport) { r.Items = map[string]Check{} }, |
| + } { |
| + r := good |
| + edit(&r) |
| + if ValidateOfflineSeal(r, host, op) == nil { |
| + t.Errorf("%s: accepted", name) |
| + } |
| + } |
| + // The evidence is the key and the hash of the bytes. |
| + ev := OfflineSealEvidence(host, op, []byte("x")) |
| + if !strings.HasPrefix(ev, OfflineSealKey(host, op)+"@") || len(ev) != len(OfflineSealKey(host, op))+1+64 { |
| + t.Errorf("evidence = %q", ev) |
| + } |
| +} |