amadan

carlos / platform Public

offline-move: console doors and status

The offline operator move's doors: start, seal, flip, open, finish and abort
under /move/offline, and the status view. Operator-only; refused with MV-O20
on a console with host claims or Phase 1 moves; every refusal comes from the
record, the edges files, the boxes' status reports and the reports under
control/offline-moves/. The only write is the record, through T1's offline
transitions.

Two pending copy rows (MV-O50, MV-O51) for the steered/unit and the
target-lists-the-host refusals.
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
claude e242e49b8c506ad52c8669327db3b378f6a0e855 parent 924ecc9
5 files changed, +2121 −2
  • approved_moves_copy_test.go +6 −2
  • docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +7 −0
  • internal/console/apiinstancemove_offline.go +977 −0
  • internal/console/apiinstancemove_offline_test.go +1128 −0
  • internal/console/authcli.go +3 −0
diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go
index d9336e645..4716b4dfe 100644
--- a/approved_moves_copy_test.go
+++ b/approved_moves_copy_test.go
@@ -96,6 +96,7 @@ var movesCopyIDs = []string{
"MV-O20", "MV-O21", "MV-O22", "MV-O23", "MV-O24", "MV-O25", "MV-O26", "MV-O27", "MV-O28", "MV-O29", "MV-O30", "MV-O31",
"MV-O32", "MV-O33",
"MV-O34", "MV-O35", "MV-O36", "MV-O37", "MV-O38", "MV-O39", "MV-O40", "MV-O41", "MV-O42", "MV-O43",
+ "MV-O50", "MV-O51",
}
// copyRow is one row of the record. The File cell is the file the sentence is
@@ -1189,6 +1190,7 @@ var movesScanFiles = []string{
"internal/host/movessources.go",
"internal/host/movesrefusals.go",
"internal/console/movesrecovery.go",
+ "internal/console/apiinstancemove_offline.go",
"internal/console/claim_runtime.go",
"internal/alerting/alerting.go",
"internal/hostops/move_log_writer.go",
@@ -1456,7 +1458,8 @@ var movesReturnedSentences = map[string]map[string]bool{
// the operator, and the literals it compares are exactly the ones allowed
// through (anything else answers one fixed sentence).
var movesLiteralFuncs = map[string][]string{
- "internal/console/apiinstancemove.go": {"phase1Conflict"},
+ "internal/console/apiinstancemove.go": {"phase1Conflict"},
+ "internal/console/apiinstancemove_offline.go": {"offlineConflict"},
}
// movesLegacyFiles are files the series touched only for plumbing (a copy
@@ -1538,7 +1541,8 @@ var movesServerText = map[string]movesServerTextSpec{
// is taken; any other return is checked as usual. Anything else answers a fixed
// sentence.
var movesMappingHelpers = map[string]map[string]bool{
- "internal/console/apiinstancemove.go": {"phase1Conflict": true},
+ "internal/console/apiinstancemove.go": {"phase1Conflict": true},
+ "internal/console/apiinstancemove_offline.go": {"offlineConflict": true},
}
// diagnosticPrefixes are the ids the readiness code puts before a reason
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
index 264d6103d..6715204bc 100644
--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
@@ -535,3 +535,10 @@ The offline transitions' refusals (fix round 1, review M4). A refusal that names
| MV-O41 | internal/provision/offline_move.go | this offline move is <state>. it finishes from finishing or reverting | pending |
| MV-O42 | internal/provision/offline_move.go | this is an offline move. use carlos move offline | pending |
| MV-O43 | internal/provision/offline_move.go | the instance is in <region>, not <from> | pending |
+
+Two more refusals of the console doors (Task 6), for cases MV-O20 to MV-O31 do not cover: the start door's class rail for a steered or unit-backed instance, and its rule that the region the instance moves to must not already list the host.
+
+| id | file | text | status |
+|---|---|---|---|
+| MV-O50 | internal/console/apiinstancemove_offline.go | <host> is steered or runs under a unit. offline moves cannot carry either yet | pending |
+| MV-O51 | internal/console/apiinstancemove_offline.go | <box> already serves <host>. an offline move needs the region it moves to to be empty of it | pending |
diff --git a/internal/console/apiinstancemove_offline.go b/internal/console/apiinstancemove_offline.go
new file mode 100644
index 000000000..32354ac35
--- /dev/null
+++ b/internal/console/apiinstancemove_offline.go
@@ -0,0 +1,977 @@
+package console
+
+import (
+ "context"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "log"
+ "net/http"
+ "os"
+ "slices"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/domains"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/status"
+ "github.com/carlosframework/platform/internal/svcrole"
+)
+
+// The offline operator move's doors (docs/superpowers/plans/2026-10-20-
+// offline-operator-move.md, Task 6). One hibernating instance moves between
+// two regions with planned downtime, an operator at each step:
+//
+// POST .../instances/{host}/move/offline {"region":"uk"} start
+// POST .../instances/{host}/move/offline/seal stopping -> sealing
+// POST .../instances/{host}/move/offline/flip sealed -> landing
+// POST .../instances/{host}/move/offline/open landed -> open
+// POST .../instances/{host}/move/offline/finish open -> finishing -> cleared
+// POST .../instances/{host}/move/offline/abort back to the source
+// GET .../instances/{host}/move/offline the status view
+//
+// Every door is moveOperatorGate (the byte-identical 404 for anyone but an
+// operator's own CLI token), then MV-O20 when this console runs host claims
+// or Phase 1 moves, then its own checks, then exactly one SetRecordMove
+// transition and a serial bump, and answers 202 with the status view.
+//
+// The doors decide from evidence the console reads and never from what the
+// operator says: the record, the edges files, the boxes' status reports and
+// the reports each box writes under control/offline-moves/. The console
+// writes nothing but the record.
+
+// offlineMoveCapability is the edges-file capability a box publishes only
+// when hibernation is configured and host claims, Phase 1 and an armed or
+// enforced guard are all off (internal/host, Task 3).
+const offlineMoveCapability = "offline-move-1"
+
+// offlineReportFresh is how recently a box's report must be written for a
+// door to believe it. Boxes rewrite their reports on every moves pass.
+const offlineReportFresh = 2 * time.Minute
+
+// Every sentence a door speaks has its row in the copy record (MV-O20 to
+// MV-O31, MV-O40, MV-O41) or is an existing approved one, written out whole
+// where it is used so the copy gate reads it.
+
+// offlineName is a region code as the operator reads it: the home's
+// configured name, or the word home when none is configured.
+func (s *Server) offlineName(code string) string {
+ if name := s.homeRegionOut(code); name != "" {
+ return name
+ }
+ return "home"
+}
+
+// offlineDoor is what every door has by the time its own checks begin.
+type offlineDoor struct {
+ acct *Account
+ app string
+ host string
+ rec *provision.Record
+ addr string
+}
+
+// offlineGate is the front half every door shares: the move family's gate
+// and operator check (a 404 for anyone else), then MV-O20.
+func (s *Server) offlineGate(w http.ResponseWriter, r *http.Request) (offlineDoor, bool) {
+ acct, app, host, rec, addr, ok := s.moveOperatorGate(w, r)
+ if !ok {
+ return offlineDoor{}, false
+ }
+ if s.cfg.HostClaim || s.cfg.MovesEnabled {
+ jsonError(w, http.StatusConflict, "offline moves are not available while host claims or Phase 1 moves are on")
+ return offlineDoor{}, false
+ }
+ return offlineDoor{acct: acct, app: app, host: host, rec: rec, addr: addr}, true
+}
+
+// offlineInFlight is the move of a door that acts on one: an offline move
+// the record carries, else MV-O24.
+func (s *Server) offlineInFlight(w http.ResponseWriter, d offlineDoor) (*provision.MoveState, bool) {
+ if d.rec.Move == nil || !d.rec.Move.Offline() {
+ jsonError(w, http.StatusConflict, fmt.Sprintf("%s has no offline move in flight", d.host))
+ return nil, false
+ }
+ return d.rec.Move, true
+}
+
+// offlineStateRefusal is MV-O25.
+func offlineStateRefusal(state, host string) *offlineRefusal {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("this move is %s. run carlos move offline status --host %s to see what it waits for", state, host)}
+}
+
+// offlineWaitRefusal is M-C10 with <from> filled in: a source box has not
+// confirmed the stop.
+func (s *Server) offlineWaitRefusal(box, from, host string) *offlineRefusal {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("Waiting for %s in %s to confirm it has stopped %s. %s is offline.", box, from, host, host)}
+}
+
+// offlineRefusal is what a door refuses with: the status and the sentence.
+type offlineRefusal struct {
+ status int
+ msg string
+}
+
+func writeRefusal(w http.ResponseWriter, ref *offlineRefusal) {
+ jsonError(w, ref.status, ref.msg)
+}
+
+// offlineConflict is the sentence for a transition's refusal. Never the
+// error's own text: phase1Conflict passes only recorded sentences and
+// answers a fixed one for anything else.
+func offlineConflict(err error) string {
+ msg := strings.TrimPrefix(err.Error(), provision.ErrMoveConflict.Error()+": ")
+ switch {
+ // The offline transitions' own recorded sentences (MV-O34 to MV-O43).
+ // Each is a whole sentence or one format filled with a state, a box or a
+ // region the record named, so it may go out as it is.
+ case msg == "this is not an offline move",
+ msg == "the record carries a different offline move",
+ msg == "this is an offline move. use carlos move offline",
+ strings.HasPrefix(msg, "a seal names the box that seals and the boxes that stopped the instance."),
+ strings.HasPrefix(msg, "this offline move is ") && strings.Contains(msg, ". this step follows "),
+ strings.HasPrefix(msg, "this offline move is ") && strings.Contains(msg, " is sealed by "),
+ strings.HasPrefix(msg, "this offline move is ") && strings.HasSuffix(msg, ". it cannot be aborted once it is open"),
+ strings.HasPrefix(msg, "this offline move is ") && strings.HasSuffix(msg, ". it finishes from finishing or reverting"),
+ strings.HasPrefix(msg, "the record's region is ") && strings.HasSuffix(msg, " the instance changed under the move"),
+ strings.HasPrefix(msg, "the instance is in ") && strings.Contains(msg, ", not "),
+ // The move family's own recorded refusals, as phase1Conflict passes
+ // them.
+ msg == "no move in flight",
+ msg == "malformed move start",
+ strings.HasPrefix(msg, "a move to ") && strings.Contains(msg, " is already "),
+ strings.HasPrefix(msg, "the instance is already in "),
+ strings.HasPrefix(msg, "a ") && strings.HasSuffix(msg, " does not move. delete it and create one in the new region"):
+ return msg
+ }
+ return "this move's state does not allow that. read its state and try again"
+}
+
+// ---- evidence ----
+
+// offlineReport is one report read from the bucket, with the bytes its
+// digest is taken over.
+type offlineReport struct {
+ moves.OfflineReport
+ raw []byte
+}
+
+// offlineEvidence is every report of one move, as read.
+type offlineEvidence struct{ reports []offlineReport }
+
+func (e offlineEvidence) get(role, box string) (offlineReport, bool) {
+ for _, r := range e.reports {
+ if r.Role == role && r.Box == box {
+ return r, true
+ }
+ }
+ return offlineReport{}, false
+}
+
+func (e offlineEvidence) byRole(role string) []offlineReport {
+ var out []offlineReport
+ for _, r := range e.reports {
+ if r.Role == role {
+ out = append(out, r)
+ }
+ }
+ return out
+}
+
+// readOfflineEvidence lists and reads every report of the move. A report
+// that does not parse, fails moves.ValidateOfflineReport, or does not sit at
+// the key its own fields name is not evidence and is skipped, which can only
+// make a door refuse. A listing or a read that fails is the error.
+func (s *Server) readOfflineEvidence(ctx context.Context, host, id string) (offlineEvidence, error) {
+ prefix := moves.OfflineReportPrefix(host, id)
+ if prefix == "" {
+ return offlineEvidence{}, nil
+ }
+ keys, err := s.cfg.Releases.List(ctx, prefix)
+ if err != nil {
+ return offlineEvidence{}, err
+ }
+ var ev offlineEvidence
+ for _, key := range keys {
+ raw, _, err := s.cfg.Releases.Get(ctx, key)
+ if errors.Is(err, os.ErrNotExist) {
+ continue
+ }
+ if err != nil {
+ return offlineEvidence{}, err
+ }
+ var rep moves.OfflineReport
+ if err := json.Unmarshal(raw, &rep); err != nil || moves.ValidateOfflineReport(rep) != nil ||
+ rep.Host != host || rep.Op != id || moves.OfflineReportKey(rep.Host, rep.Op, rep.Role, rep.Box) != key {
+ log.Printf("console: offline move %s: ignoring %s: not a report of this move", host, key)
+ continue
+ }
+ ev.reports = append(ev.reports, offlineReport{OfflineReport: rep, raw: raw})
+ }
+ sort.Slice(ev.reports, func(i, j int) bool {
+ if ev.reports[i].Role != ev.reports[j].Role {
+ return ev.reports[i].Role < ev.reports[j].Role
+ }
+ return ev.reports[i].Box < ev.reports[j].Box
+ })
+ return ev, nil
+}
+
+func offlineFresh(r moves.OfflineReport, now time.Time) bool {
+ age := now.Sub(r.At)
+ return !r.At.IsZero() && age <= offlineReportFresh && age >= -time.Minute
+}
+
+func offlineStopped(r moves.OfflineReport) bool { return r.Held && r.Fenced && r.Quiesced }
+
+// offlineSealItems and offlineLandingItems are the one place the doors ask
+// the manifest which items a report must carry OK: flip needs the sealer's
+// report OK for every item the source side reports (Source "seal"), open the
+// target's for every item the landing owns.
+func offlineSealItems(r moves.OfflineReport) (bool, []string) {
+ return r.SourceItemsOK()
+}
+
+func offlineLandingItems(r moves.OfflineReport) (bool, []string) {
+ return r.ItemsOK(moves.OfflineOwnerLanding)
+}
+
+// offlineEvidenceMatches says whether the record's source.evidence names
+// this report: the report's sha256 as hex, bare or after the report's key
+// and one separator.
+func offlineEvidenceMatches(evidence, key string, raw []byte) bool {
+ sum := sha256.Sum256(raw)
+ h := hex.EncodeToString(sum[:])
+ tail, ok := strings.CutPrefix(evidence, key)
+ if ok && tail != "" && strings.Contains("@: #", tail[:1]) {
+ evidence = tail[1:]
+ }
+ return evidence == h || evidence == "sha256:"+h
+}
+
+// offlineFleet is the edges files and each box's status report, read
+// together. Incomplete evidence is not an answer: ok is false.
+type offlineFleet struct {
+ snap edgesSnapshot
+ metrics map[string]*status.BoxMetrics
+}
+
+func (s *Server) readOfflineFleet(ctx context.Context) (offlineFleet, bool) {
+ snap, err := s.liveEdges(ctx)
+ if err != nil || !snap.EvidenceComplete() {
+ return offlineFleet{}, false
+ }
+ boxes, complete, err := status.LoadBoxesComplete(ctx, s.cfg.Releases)
+ if err != nil || !complete {
+ return offlineFleet{}, false
+ }
+ f := offlineFleet{snap: snap, metrics: map[string]*status.BoxMetrics{}}
+ for _, b := range boxes {
+ f.metrics[b.Label] = b.Metrics
+ }
+ sort.Slice(f.snap.Files, func(i, j int) bool { return f.snap.Files[i].Box < f.snap.Files[j].Box })
+ return f, true
+}
+
+// offlineCanHold says whether a box's published state lets it hold an
+// ordinary instance: hibernation not configured off, and the shared pool not
+// served off. Absent is unknown, and unknown could hold.
+func offlineCanHold(f edgesFilePlacementView) bool {
+ return (f.HibernationConfigured == nil || *f.HibernationConfigured) &&
+ (f.ServesSharedPool == nil || *f.ServesSharedPool)
+}
+
+func offlineCapable(f edgesFilePlacementView) bool {
+ return slices.Contains(f.Capabilities, offlineMoveCapability)
+}
+
+func offlineServes(f edgesFilePlacementView, host string) bool {
+ _, ok := f.Hosts[host]
+ return ok
+}
+
+// offlineClaimsOnBox is what a claims-on box has to show the start door
+// (ruling 2026-10-20, point 3). Every field is published state; none is the
+// operator's word. Census and SharedPool are pointers: nil is "the box does
+// not say", and an unsaid fact refuses.
+type offlineClaimsOnBox struct {
+ HostRow bool // its edges file lists the host
+ Census *bool // its report says root's census.json is present
+ SharedPool *bool // its edges file says whether it serves the shared pool
+}
+
+// ok: no row for the host, census.json reported present, and pools that do
+// not match the record's placement (the record is in the shared pool, so a
+// box that serves it matches).
+func (b offlineClaimsOnBox) ok() bool {
+ return !b.HostRow && b.Census != nil && *b.Census && b.SharedPool != nil && !*b.SharedPool
+}
+
+// offlineCensusReported is whether a box's status report says root's
+// census.json is present. No report field carries it today, and the move
+// adds none, so every claims-on box fails this condition until one does.
+func offlineCensusReported(*status.BoxMetrics) *bool { return nil }
+
+// offlineSourceBoxes are the edges files of the record's source placement
+// that could hold the host, in box order.
+func (f offlineFleet) boxesAt(code string) []edgesFilePlacementView {
+ var out []edgesFilePlacementView
+ for _, b := range f.snap.Files {
+ if b.Placement == code && offlineCanHold(b) {
+ out = append(out, b)
+ }
+ }
+ return out
+}
+
+// ---- start ----
+
+// offlineSelect is the start door's box rules, from the fleet read and
+// nothing else: exactly one target box, at least one source box, the
+// capability on each, no claims-on box in either region that could touch the
+// host, an empty target, the deployment's secrets key on the target and a
+// published data bucket there.
+func (s *Server) offlineSelect(ctx context.Context, f offlineFleet, host, from, to string) *offlineRefusal {
+ fromName, toName := s.offlineName(from), s.offlineName(to)
+ src, tgt := f.boxesAt(from), f.boxesAt(to)
+ if len(tgt) != 1 {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s has %d boxes. an offline move needs exactly one box in the region it moves to", toName, len(tgt))}
+ }
+ if len(src) == 0 {
+ return &offlineRefusal{status: http.StatusBadRequest, msg: "no eligible box in " + fromName + " publishes an edges file"}
+ }
+ for _, b := range append(slices.Clone(src), tgt...) {
+ if !offlineCapable(b) {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s cannot take part in an offline move. it runs an older release or has host claims on", b.Box)}
+ }
+ }
+ // Ruling 2026-10-20, point 3: a box in either region that runs claims, or
+ // whose report does not say it does not, must publish no row for the
+ // host, report census.json present and serve pools the record's
+ // placement does not match. A box with the capability has vouched for
+ // itself (it publishes it only with claims off) unless its report says
+ // claims are on.
+ for _, b := range f.snap.Files {
+ if b.Placement != from && b.Placement != to {
+ continue
+ }
+ m := f.metrics[b.Box]
+ if (offlineCapable(b) && (m == nil || !m.HostClaimEnabled)) || (m != nil && !m.HostClaimEnabled) {
+ continue
+ }
+ if !(offlineClaimsOnBox{HostRow: offlineServes(b, host), Census: offlineCensusReported(m), SharedPool: b.ServesSharedPool}).ok() {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s cannot take part in an offline move. it runs an older release or has host claims on", b.Box)}
+ }
+ }
+ t := tgt[0]
+ if offlineServes(t, host) {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s already serves %s. an offline move needs the region it moves to to be empty of it", t.Box, host)}
+ }
+ if fp := s.expectedSecretsFP(); fp == "" || t.SecretsKeyFP != fp {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s does not hold this deployment's secrets key", t.Box)}
+ }
+ if t.DataBucket == "" {
+ caps, _, _ := movePlacements(ctx, s.cfg.Releases, s.moveHomeName())
+ return &offlineRefusal{status: http.StatusBadRequest, msg: "region " + toName + " cannot take a move in this deployment. it publishes no staging bucket. Regions that can: " + s.stagingNames(caps)}
+ }
+ return nil
+}
+
+// offlineClaimEvidence refuses a host that has, anywhere, a claim: the
+// record's acquisition, control/hosts/<host>.json, a domains claim carrying
+// one, or the app's admission (MV-O29); and a deployment that has begun the
+// claim backfill (MV-O20). A read that fails is a 502 and never an absence.
+func (s *Server) offlineClaimEvidence(ctx context.Context, d offlineDoor) *offlineRefusal {
+ has := &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s has a host claim. offline moves are only for hosts without one", d.host)}
+ unreadable := &offlineRefusal{status: http.StatusBadGateway, msg: "could not read the host claim"}
+ if d.rec.Acquisition != "" {
+ return has
+ }
+ obs, err := hostclaim.Read(ctx, s.cfg.Releases, d.host)
+ if err != nil {
+ log.Printf("console: offline move %s: claim read: %v", d.host, err)
+ return unreadable
+ }
+ if !obs.Absent {
+ return has
+ }
+ if c, err := domains.GetClaim(ctx, s.cfg.Releases, d.host); err == nil && c.Acquisition != "" {
+ return has
+ } else if err != nil && !errors.Is(err, os.ErrNotExist) {
+ log.Printf("console: offline move %s: domains claim read: %v", d.host, err)
+ return unreadable
+ }
+ key, err := hostops.AdmissionKey(d.acct.Sqid, d.app)
+ if err != nil {
+ log.Printf("console: offline move %s: admission key: %v", d.host, err)
+ return unreadable
+ }
+ if _, _, err := s.cfg.Releases.Get(ctx, key); err == nil {
+ return has
+ } else if !errors.Is(err, os.ErrNotExist) {
+ log.Printf("console: offline move %s: admission read: %v", d.host, err)
+ return unreadable
+ }
+ // A plan at done is allowed; one that has begun, or that cannot be
+ // verified, is not.
+ switch err := hostops.CheckCreateFreeze(ctx, s.cfg.Releases, s.cfg.ReleasePub); {
+ case err == nil:
+ case errors.Is(err, hostops.ErrPending), errors.Is(err, hostops.ErrIdentity):
+ return &offlineRefusal{status: http.StatusConflict, msg: "offline moves are not available while host claims or Phase 1 moves are on"}
+ default:
+ log.Printf("console: offline move %s: backfill plan read: %v", d.host, err)
+ return unreadable
+ }
+ return nil
+}
+
+func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Request) {
+ d, ok := s.offlineGate(w, r)
+ if !ok {
+ return
+ }
+ rec, host := d.rec, d.host
+ if s.refuseSeedMove(w, rec) {
+ return
+ }
+ if rec.Role != "" {
+ jsonError(w, http.StatusBadRequest, "a replica does not move. delete it and create one in the new region")
+ return
+ }
+ if len(rec.Replicas) > 0 {
+ jsonError(w, http.StatusBadRequest, "a home with replicas does not move. remove its replicas first")
+ return
+ }
+ if rec.Steering != "" || rec.Unit != "" {
+ jsonError(w, http.StatusBadRequest, fmt.Sprintf("%s is steered or runs under a unit. offline moves cannot carry either yet", host))
+ return
+ }
+ ctx := r.Context()
+ if ref := s.offlineClaimEvidence(ctx, d); ref != nil {
+ writeRefusal(w, ref)
+ return
+ }
+ all, unreadable, err := provision.ListAllRecordsStrict(ctx, s.cfg.Releases)
+ if err != nil || len(unreadable) > 0 {
+ log.Printf("console: offline move %s: listing records: %v (%d unreadable)", host, err, len(unreadable))
+ jsonError(w, http.StatusBadGateway, "could not read the instance")
+ return
+ }
+ for _, o := range all {
+ if o.Host != host && o.Move.Offline() {
+ jsonError(w, http.StatusConflict, fmt.Sprintf("%s is moving offline. move one instance at a time", o.Host))
+ return
+ }
+ }
+ var req struct {
+ Region string `json:"region"`
+ }
+ if !readJSON(w, r, &req) {
+ return
+ }
+ region := strings.TrimSpace(req.Region)
+ to := s.homeRegionIn(region)
+ if region == "" || !provision.ValidRegion(to) {
+ jsonError(w, http.StatusBadRequest, "invalid region")
+ return
+ }
+ if to == rec.Region {
+ jsonError(w, http.StatusConflict, "the instance is already in "+region)
+ return
+ }
+ m, err := provision.GetManifest(ctx, s.cfg.Releases, d.acct.Sqid, d.app)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ log.Printf("console: offline move %s: manifest: %v", host, err)
+ jsonError(w, http.StatusServiceUnavailable, "could not read the app's instances manifest. try again")
+ return
+ }
+ if m != nil && !s.moveRegionAllowed(m.Regions, to) {
+ jsonError(w, http.StatusBadRequest, "region "+region+" is not one of this app's regions: "+strings.Join(m.Regions, ", "))
+ return
+ }
+ fleet, ok := s.readOfflineFleet(ctx)
+ if !ok {
+ jsonError(w, http.StatusConflict, "the fleet's edges files could not all be read. try again")
+ return
+ }
+ if to == "" {
+ // A box that sets CARLOS_REGION to the home's name is home to this
+ // console and not to itself, so it cannot take the move.
+ for _, f := range fleet.snap.Files {
+ if f.Placement != "" && homePlacementCode(s.moveHomeName(), f.Placement) == "" {
+ jsonError(w, http.StatusConflict, f.Box+" sets CARLOS_REGION, so it cannot take a move into the home region")
+ return
+ }
+ }
+ }
+ if ref := s.offlineSelect(ctx, fleet, host, rec.Region, to); ref != nil {
+ writeRefusal(w, ref)
+ return
+ }
+ id, err := moves.NewMoveID(rand.Reader)
+ if err != nil {
+ log.Printf("console: offline move %s: move id: %v", host, err)
+ jsonError(w, http.StatusBadGateway, "could not mint a move id")
+ return
+ }
+ start := provision.OfflineStart(id, rec.Region, to, d.addr, time.Now())
+ probe := *rec
+ if err := start(&probe); err != nil {
+ jsonError(w, http.StatusConflict, offlineConflict(err))
+ return
+ }
+ s.offlineAdvance(w, r, d, start)
+}
+
+// offlineAdvance applies one transition and answers 202 with the status
+// view of the record it left.
+func (s *Server) offlineAdvance(w http.ResponseWriter, r *http.Request, d offlineDoor, mut func(*provision.Record) error) {
+ ctx := r.Context()
+ rec, err := provision.SetRecordMove(ctx, s.cfg.Releases, d.acct.Sqid, d.app, d.host, mut)
+ if errors.Is(err, provision.ErrMoveConflict) {
+ jsonError(w, http.StatusConflict, offlineConflict(err))
+ return
+ }
+ if err != nil {
+ log.Printf("console: offline move %s/%s/%s: %v", d.acct.Sqid, d.app, d.host, err)
+ jsonError(w, http.StatusBadGateway, "could not write the move. try again")
+ return
+ }
+ s.bumpSerial(ctx)
+ writeJSON(w, http.StatusAccepted, s.offlineViewOf(ctx, rec))
+}
+
+// ---- the step each state waits for ----
+
+// The verbs.
+const (
+ offlineVerbSeal = "seal"
+ offlineVerbFlip = "flip"
+ offlineVerbOpen = "open"
+ offlineVerbFinish = "finish"
+)
+
+// offlineVerbOf is the one door a state is waiting on. sealing and landing
+// wait for a box, not for the operator.
+func offlineVerbOf(state string) string {
+ switch state {
+ case provision.OfflineStopping:
+ return offlineVerbSeal
+ case provision.OfflineSealed:
+ return offlineVerbFlip
+ case provision.OfflineLanded:
+ return offlineVerbOpen
+ case provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting:
+ return offlineVerbFinish
+ }
+ return ""
+}
+
+// offlineStep is a door's checks passed: the transition to write.
+type offlineStep struct {
+ mut func(*provision.Record) error
+}
+
+// offlineOpenMut is the open transition. The one place the door names it.
+func offlineOpenMut(id string, now time.Time) func(*provision.Record) error {
+ return provision.OfflineOpenTransition(id, now)
+}
+
+// offlineStepFor runs the checks of the verb the move's state waits for,
+// from the evidence. A refusal is the sentence the door answers and the
+// status view shows as what the move waits for.
+func (s *Server) offlineStepFor(ctx context.Context, d offlineDoor, mv *provision.MoveState, ev offlineEvidence, now time.Time) (offlineStep, *offlineRefusal) {
+ host, from := d.host, s.offlineName(mv.From)
+ switch mv.State {
+ case provision.OfflineStopping:
+ fleet, ok := s.readOfflineFleet(ctx)
+ if !ok {
+ return offlineStep{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again"}
+ }
+ sealer, boxes, ref := s.offlineSealCheck(fleet, ev, mv, host, from, now)
+ if ref != nil {
+ return offlineStep{}, ref
+ }
+ return offlineStep{mut: provision.OfflineSeal(mv.ID, sealer, boxes)}, nil
+ case provision.OfflineSealed:
+ if ref := s.offlineFlipCheck(ev, mv, host, from, now); ref != nil {
+ return offlineStep{}, ref
+ }
+ return offlineStep{mut: provision.OfflineFlip(mv.ID)}, nil
+ case provision.OfflineLanded:
+ if ref := s.offlineOpenCheck(ev, mv, host, from, now); ref != nil {
+ return offlineStep{}, ref
+ }
+ return offlineStep{mut: offlineOpenMut(mv.ID, now)}, nil
+ case provision.OfflineOpen:
+ if ref := offlineFinishCheck(ev, mv, host, moves.OfflinePhaseOpened, "", moves.OfflinePhaseParked, mv.Boxes); ref != nil {
+ return offlineStep{}, ref
+ }
+ return offlineStep{mut: provision.OfflineFinishingTransition(mv.ID)}, nil
+ case provision.OfflineFinishing:
+ if ref := offlineFinishCheck(ev, mv, host, "", moves.OfflinePhaseLifted, moves.OfflinePhaseGone, mv.Boxes); ref != nil {
+ return offlineStep{}, ref
+ }
+ return offlineStep{mut: provision.OfflineFinish(mv.ID, now)}, nil
+ case provision.OfflineReverting:
+ // Every box the seal named, and the sealer. A move aborted before
+ // the seal named neither: the boxes are the source region's as the
+ // edges files publish them now, and there is no sealer (it never
+ // took a tombstone).
+ boxes, sealerPhase := mv.Boxes, moves.OfflinePhaseReverted
+ if mv.Sealer == "" {
+ fleet, ok := s.readOfflineFleet(ctx)
+ if !ok {
+ return offlineStep{}, &offlineRefusal{status: http.StatusConflict, msg: "the fleet's edges files could not all be read. try again"}
+ }
+ boxes, sealerPhase = nil, ""
+ for _, b := range fleet.boxesAt(mv.From) {
+ boxes = append(boxes, b.Box)
+ }
+ }
+ if ref := offlineFinishCheck(ev, mv, host, "", sealerPhase, moves.OfflinePhaseReverted, boxes); ref != nil {
+ return offlineStep{}, ref
+ }
+ return offlineStep{mut: provision.OfflineFinish(mv.ID, now)}, nil
+ }
+ return offlineStep{}, offlineStateRefusal(mv.State, host)
+}
+
+// offlineSealCheck: every source box reports this move held, fenced and
+// quiesced, and its edges file no longer lists the host. It names the sealer
+// and the boxes the later doors wait for.
+func (s *Server) offlineSealCheck(f offlineFleet, ev offlineEvidence, mv *provision.MoveState, host, from string, now time.Time) (sealer string, boxes []string, ref *offlineRefusal) {
+ src := f.boxesAt(mv.From)
+ if len(src) == 0 {
+ return "", nil, &offlineRefusal{status: http.StatusBadRequest, msg: "no eligible box in " + from + " publishes an edges file"}
+ }
+ var data []string
+ for _, b := range src {
+ rep, ok := ev.get(moves.OfflineRoleSource, b.Box)
+ if !ok || !offlineFresh(rep.OfflineReport, now) || !offlineStopped(rep.OfflineReport) || offlineServes(b, host) {
+ return "", nil, s.offlineWaitRefusal(b.Box, from, host)
+ }
+ boxes = append(boxes, b.Box)
+ if rep.LocalDB || rep.Files || rep.Env {
+ data = append(data, b.Box)
+ }
+ }
+ for _, b := range boxes {
+ if rep, _ := ev.get(moves.OfflineRoleSource, b); rep.Sidecar {
+ return "", nil, &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s keeps a sidecar file for %s. instances with sidecars cannot move yet", b, host)}
+ }
+ }
+ if len(data) > 1 {
+ return "", nil, &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s keeps local data on more than one box: %s. remove the stale copy first", host, strings.Join(data, ", "))}
+ }
+ sealer = boxes[0]
+ if len(data) == 1 {
+ sealer = data[0]
+ }
+ return sealer, boxes, nil
+}
+
+// offlineFlipCheck: the seal's report is the one the record names and every
+// item it owns is OK; every source box has stopped and left no process or
+// layout journal, on a report this fresh.
+func (s *Server) offlineFlipCheck(ev offlineEvidence, mv *provision.MoveState, host, from string, now time.Time) *offlineRefusal {
+ seal, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer)
+ if !ok || mv.Source == nil || !offlineEvidenceMatches(mv.Source.Evidence, moves.OfflineReportKey(host, mv.ID, moves.OfflineRoleSealer, mv.Sealer), seal.raw) {
+ return offlineStateRefusal(mv.State, host)
+ }
+ if items, _ := offlineSealItems(seal.OfflineReport); !items {
+ return offlineStateRefusal(mv.State, host)
+ }
+ for _, box := range mv.Boxes {
+ rep, ok := ev.get(moves.OfflineRoleSource, box)
+ if !ok || !offlineFresh(rep.OfflineReport, now) {
+ return processRefusal(box, host)
+ }
+ if !offlineStopped(rep.OfflineReport) {
+ return s.offlineWaitRefusal(box, from, host)
+ }
+ if rep.Procs != 0 || rep.LayoutJournal {
+ return processRefusal(box, host)
+ }
+ }
+ return nil
+}
+
+func processRefusal(box, host string) *offlineRefusal {
+ return &offlineRefusal{status: http.StatusConflict, msg: fmt.Sprintf("%s still runs a process for %s. wait for it to stop, then flip", box, host)}
+}
+
+// offlineOpenCheck is the re-check of the stop: the target's landing report
+// has every landing item OK, the sealer's report is from this pass and holds
+// the tombstone for this move, and every source box is still held, fenced and
+// quiesced.
+func (s *Server) offlineOpenCheck(ev offlineEvidence, mv *provision.MoveState, host, from string, now time.Time) *offlineRefusal {
+ tgt := ev.byRole(moves.OfflineRoleTarget)
+ if len(tgt) != 1 {
+ return offlineStateRefusal(mv.State, host)
+ }
+ if items, _ := offlineLandingItems(tgt[0].OfflineReport); !items {
+ return offlineStateRefusal(mv.State, host)
+ }
+ seal, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer)
+ if !ok || !offlineFresh(seal.OfflineReport, now) || seal.TombstoneOwner != moves.TombstoneOwner(mv.ID) || !seal.TombstoneUntil.After(now) {
+ return offlineStateRefusal(mv.State, host)
+ }
+ for _, box := range mv.Boxes {
+ rep, ok := ev.get(moves.OfflineRoleSource, box)
+ if !ok || !offlineFresh(rep.OfflineReport, now) || !offlineStopped(rep.OfflineReport) {
+ return s.offlineWaitRefusal(box, from, host)
+ }
+ }
+ return nil
+}
+
+// offlineFinishCheck: the target's phase (when targetPhase is set), the
+// sealer's phase (when sealerPhase is set) and every source box's phase,
+// each as the box's own report says. Otherwise MV-O25. Open to finishing
+// wants the target opened and every source parked; finishing to cleared the
+// sealer lifted and every source gone; reverting to cleared the sealer and
+// every source reverted (the target is not asked: a target box that died
+// after the flip must not hold the abort).
+func offlineFinishCheck(ev offlineEvidence, mv *provision.MoveState, host, targetPhase, sealerPhase, sourcePhase string, boxes []string) *offlineRefusal {
+ refuse := offlineStateRefusal(mv.State, host)
+ if targetPhase != "" {
+ tgt := ev.byRole(moves.OfflineRoleTarget)
+ if len(tgt) != 1 || tgt[0].Phase != targetPhase {
+ return refuse
+ }
+ }
+ if sealerPhase != "" {
+ if rep, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer); !ok || rep.Phase != sealerPhase {
+ return refuse
+ }
+ }
+ if len(boxes) == 0 {
+ return refuse
+ }
+ for _, box := range boxes {
+ if rep, ok := ev.get(moves.OfflineRoleSource, box); !ok || rep.Phase != sourcePhase {
+ return refuse
+ }
+ }
+ return nil
+}
+
+// offlineVerbDoor is the shape of seal, flip, open and finish: the gate, an
+// offline move in flight, the state the verb follows, the evidence, the
+// verb's checks, one transition.
+func (s *Server) offlineVerbDoor(verb string) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ d, ok := s.offlineGate(w, r)
+ if !ok {
+ return
+ }
+ mv, ok := s.offlineInFlight(w, d)
+ if !ok {
+ return
+ }
+ if offlineVerbOf(mv.State) != verb {
+ writeRefusal(w, offlineStateRefusal(mv.State, d.host))
+ return
+ }
+ ctx := r.Context()
+ ev, err := s.readOfflineEvidence(ctx, d.host, mv.ID)
+ if err != nil {
+ log.Printf("console: offline move %s: reports: %v", d.host, err)
+ jsonError(w, http.StatusBadGateway, "could not read the move's reports. try again")
+ return
+ }
+ step, ref := s.offlineStepFor(ctx, d, mv, ev, time.Now())
+ if ref != nil {
+ writeRefusal(w, ref)
+ return
+ }
+ s.offlineAdvance(w, r, d, step.mut)
+ }
+}
+
+func (s *Server) handleAPIInstanceOfflineAbort(w http.ResponseWriter, r *http.Request) {
+ d, ok := s.offlineGate(w, r)
+ if !ok {
+ return
+ }
+ mv, ok := s.offlineInFlight(w, d)
+ if !ok {
+ return
+ }
+ switch mv.State {
+ case provision.OfflineOpen, provision.OfflineFinishing:
+ jsonError(w, http.StatusConflict, fmt.Sprintf("%s is already served from %s. move it back with a new move", d.host, s.offlineName(mv.To)))
+ return
+ case provision.OfflineReverting:
+ // Already reverting: nothing to write.
+ writeJSON(w, http.StatusAccepted, s.offlineViewOf(r.Context(), d.rec))
+ return
+ }
+ s.offlineAdvance(w, r, d, provision.OfflineAbort(mv.ID))
+}
+
+// ---- status ----
+
+// offlineReportView is a report as the status view carries it: every field
+// the box wrote, and whether it is current enough for a door to believe.
+type offlineReportView struct {
+ moves.OfflineReport
+ Fresh bool `json:"fresh"`
+}
+
+// offlineItemView is one row of the manifest with the latest check of it.
+// Status is ok, waiting, failed or unchecked (no report decides the item).
+type offlineItemView struct {
+ ID string `json:"id"`
+ Item string `json:"item"`
+ Disposition string `json:"disposition"`
+ Owner string `json:"owner"`
+ Source string `json:"source,omitempty"`
+ Status string `json:"status"`
+ Detail string `json:"detail,omitempty"`
+}
+
+// offlineView is the status answer, and the 202 every mutating door gives.
+// State is "none" when no offline move is in flight (Last then carries the
+// outcome of the one that ended). Next is the verb that moves the move on
+// right now; empty while a box has work to do. Waiting is the sentence that
+// verb would be refused with. CanAbort says abort is legal.
+type offlineView struct {
+ Host string `json:"host"`
+ State string `json:"state"`
+ ID string `json:"id,omitempty"`
+ From string `json:"from,omitempty"`
+ To string `json:"to,omitempty"`
+ Sealer string `json:"sealer,omitempty"`
+ Boxes []string `json:"boxes,omitempty"`
+ RequestedBy string `json:"requested_by,omitempty"`
+ RequestedAt int64 `json:"requested_at,omitempty"`
+ UpdatedAt int64 `json:"updated_at,omitempty"`
+ OpenedAt int64 `json:"opened_at,omitempty"`
+ Next string `json:"next,omitempty"`
+ Waiting string `json:"waiting,omitempty"`
+ CanAbort bool `json:"can_abort,omitempty"`
+ Reports []offlineReportView `json:"reports,omitempty"`
+ Items []offlineItemView `json:"items,omitempty"`
+ Last *provision.MoveOutcome `json:"last,omitempty"`
+}
+
+// offlineViewOf builds the status view of a record. Evidence it cannot read
+// leaves the reports out and says so in Waiting; it never fails the answer.
+func (s *Server) offlineViewOf(ctx context.Context, rec *provision.Record) offlineView {
+ v := offlineView{Host: rec.Host, State: "none", Last: rec.LastMove}
+ mv := rec.Move
+ if mv == nil || !mv.Offline() {
+ return v
+ }
+ now := time.Now()
+ v.State, v.ID = mv.State, mv.ID
+ v.From, v.To = s.offlineName(mv.From), s.offlineName(mv.To)
+ v.Sealer, v.Boxes = mv.Sealer, mv.Boxes
+ v.RequestedBy, v.RequestedAt, v.UpdatedAt = mv.RequestedBy, mv.RequestedAt.Unix(), mv.UpdatedAt.Unix()
+ if mv.OpenedAt != nil {
+ v.OpenedAt = mv.OpenedAt.Unix()
+ }
+ switch mv.State {
+ case provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding, provision.OfflineLanded:
+ v.CanAbort = true
+ }
+ ev, err := s.readOfflineEvidence(ctx, rec.Host, mv.ID)
+ if err != nil {
+ log.Printf("console: offline move %s: reports: %v", rec.Host, err)
+ v.Waiting = "could not read the move's reports. try again"
+ return v
+ }
+ for _, r := range ev.reports {
+ v.Reports = append(v.Reports, offlineReportView{OfflineReport: r.OfflineReport, Fresh: offlineFresh(r.OfflineReport, now)})
+ }
+ v.Items = offlineItems(ev, mv)
+ if verb := offlineVerbOf(mv.State); verb != "" {
+ d := offlineDoor{host: rec.Host}
+ if _, ref := s.offlineStepFor(ctx, d, mv, ev, now); ref != nil {
+ v.Waiting = ref.msg
+ } else {
+ v.Next = verb
+ }
+ }
+ // Landing waits for a box, unless the box says it cannot land.
+ if mv.State == provision.OfflineLanding {
+ for _, r := range ev.byRole(moves.OfflineRoleTarget) {
+ if r.Phase == moves.OfflinePhaseFailed {
+ v.Waiting = fmt.Sprintf("%s could not land %s. run carlos move offline abort --host %s", r.Box, rec.Host, rec.Host)
+ }
+ }
+ }
+ return v
+}
+
+// offlineItems is the manifest with each item's latest check, from the
+// report of the step that owns it: the sealer's for a seal item, the
+// target's landing report for a landing item. A check present and not OK is
+// failed when the report that holds it says failed, else waiting (the box
+// retries it). An item no report decides is unchecked, and a door-owned one
+// was decided at start.
+func offlineItems(ev offlineEvidence, mv *provision.MoveState) []offlineItemView {
+ owners := map[string][]offlineReport{moves.OfflineOwnerLanding: ev.byRole(moves.OfflineRoleTarget)}
+ if r, ok := ev.get(moves.OfflineRoleSealer, mv.Sealer); ok {
+ owners[moves.OfflineOwnerSeal] = []offlineReport{r}
+ }
+ out := make([]offlineItemView, 0, len(moves.OfflineManifest))
+ for _, it := range moves.OfflineManifest {
+ row := offlineItemView{ID: it.ID, Item: it.Item, Disposition: it.Disposition, Owner: it.Owner, Source: it.Source, Status: "waiting"}
+ switch it.Owner {
+ case moves.OfflineOwnerDoor:
+ row.Status = "ok"
+ case moves.OfflineOwnerNone:
+ row.Status = "unchecked"
+ default:
+ for _, r := range owners[it.Owner] {
+ c, found := r.Items[it.ID]
+ if !found {
+ continue
+ }
+ row.Detail = c.Reason
+ switch {
+ case c.OK:
+ row.Status = "ok"
+ case r.Phase == moves.OfflinePhaseFailed:
+ row.Status = "failed"
+ }
+ break
+ }
+ }
+ out = append(out, row)
+ }
+ return out
+}
+
+func (s *Server) handleAPIInstanceOfflineStatus(w http.ResponseWriter, r *http.Request) {
+ d, ok := s.offlineGate(w, r)
+ if !ok {
+ return
+ }
+ writeJSON(w, http.StatusOK, s.offlineViewOf(r.Context(), d.rec))
+}
+
+// registerOfflineMoveRoutes registers the offline move's doors. They are
+// always registered: a console on the claim path or with Phase 1 on answers
+// every one with MV-O20, after the operator gate.
+func (s *Server) registerOfflineMoveRoutes() {
+ const base = "/api/cli/apps/{account}/{app}/instances/{host}/move/offline"
+ inst := svcrole.Allow(svcrole.Instance)
+ s.cliRoute("GET "+base, inst, s.handleAPIInstanceOfflineStatus)
+ s.cliRoute("POST "+base, inst, s.handleAPIInstanceOfflineStart)
+ s.cliRoute("POST "+base+"/seal", inst, s.offlineVerbDoor(offlineVerbSeal))
+ s.cliRoute("POST "+base+"/flip", inst, s.offlineVerbDoor(offlineVerbFlip))
+ s.cliRoute("POST "+base+"/open", inst, s.offlineVerbDoor(offlineVerbOpen))
+ s.cliRoute("POST "+base+"/finish", inst, s.offlineVerbDoor(offlineVerbFinish))
+ s.cliRoute("POST "+base+"/abort", inst, s.handleAPIInstanceOfflineAbort)
+}
diff --git a/internal/console/apiinstancemove_offline_test.go b/internal/console/apiinstancemove_offline_test.go
new file mode 100644
index 000000000..49a7776e8
--- /dev/null
+++ b/internal/console/apiinstancemove_offline_test.go
@@ -0,0 +1,1128 @@
+package console
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/domains"
+ "github.com/carlosframework/platform/internal/hostops"
+ "github.com/carlosframework/platform/internal/moves"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+ "github.com/carlosframework/platform/internal/svcrole"
+)
+
+// The offline operator move's doors, driven in-process against the same
+// console fixture the move doors' tests use. The boxes' side is played by
+// reports written at control/offline-moves/ exactly where the real boxes
+// write them (moves.OfflineReportKey); the record moves only through
+// provision's offline transitions.
+
+const offlineHost = "acme.go.tito.io"
+
+// offlineBox is one box the fleet publishes: an edges file and a status
+// snapshot.
+type offlineBox struct {
+ Box, Placement, Bucket string
+ NoCap bool // no offline-move-1
+ ClaimsOn bool // its status report says claims are on
+ NoMetrics bool // its status report carries no metrics
+ NotSharedPool bool // serves_shared_pool false
+ NoHibernation bool // hibernation_configured false
+ Hosts []string // hosts its edges file lists
+ FP string // secrets_key_fp; testKeyFP when empty
+}
+
+func ieOff(box string) offlineBox { return offlineBox{Box: box, Placement: "ie", Bucket: "data-ie"} }
+func ukOff(box string) offlineBox { return offlineBox{Box: box, Placement: "uk", Bucket: "data-uk"} }
+
+type offlineWorld struct {
+ t *testing.T
+ fx *consoleFixture
+ token string
+ base string // .../instances/<host>/move/offline
+ id string // the started move's id
+}
+
+// newOfflineWorld is an operator console with one hibernating instance in
+// region ie and the fleet published.
+func newOfflineWorld(t *testing.T, boxes ...offlineBox) *offlineWorld {
+ t.Helper()
+ return newOfflineWorldIn(t, "", "ie", boxes...)
+}
+
+// newOfflineWorldIn is newOfflineWorld with the console's home named and the
+// instance created in the given region.
+func newOfflineWorldIn(t *testing.T, home, region string, boxes ...offlineBox) *offlineWorld {
+ t.Helper()
+ fx, token := instanceFixture(t, "code-offline-"+strings.ReplaceAll(t.Name(), "/", "-"))
+ if err := fx.st.SetOperator("alice@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
+ fx.srv.cfg.HomeRegion = home
+ m := titoManifest()
+ m.Regions = nil
+ m.Pool = ""
+ enableInstances(t, fx, "titogo", m)
+ u := instancesURL(fx, "bab", "titogo")
+ if c, _, out := rawReq(t, http.MethodPost, u, token, `{"host":"`+offlineHost+`","region":"`+region+`"}`); c != http.StatusCreated {
+ t.Fatalf("create = %d %s", c, out)
+ }
+ w := &offlineWorld{t: t, fx: fx, token: token, base: u + "/" + offlineHost + "/move/offline"}
+ w.fleet(boxes...)
+ return w
+}
+
+// putRaw writes a bucket object as a fixture, replacing what is there.
+func offlinePut(t *testing.T, rel release.Store, key string, body []byte) {
+ t.Helper()
+ _, etag, err := rel.Get(context.Background(), key)
+ if err != nil {
+ etag = ""
+ }
+ if _, err := rel.Put(context.Background(), key, body, etag); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// fleet publishes the boxes, replacing any earlier publication of the same
+// box.
+func (w *offlineWorld) fleet(boxes ...offlineBox) {
+ w.t.Helper()
+ w.fx.srv.cfg.SecretsPub = testKeyPub
+ for _, b := range boxes {
+ aws := "eu-west-1"
+ if b.Placement == "uk" {
+ aws = "eu-west-2"
+ }
+ caps := []string{"region-ha-1"}
+ if !b.NoCap {
+ caps = append(caps, offlineMoveCapability)
+ }
+ hosts := map[string]any{}
+ for _, h := range b.Hosts {
+ hosts[h] = map[string]any{"role": "home"}
+ }
+ fp := b.FP
+ if fp == "" {
+ fp = testKeyFP
+ }
+ body, err := json.Marshal(map[string]any{
+ "v": 1, "box": b.Box, "box_id": b.Box, "eip": "1.2.3.4", "region": aws, "placement": b.Placement,
+ "data_bucket": b.Bucket, "hibernation_configured": !b.NoHibernation, "serves_shared_pool": !b.NotSharedPool,
+ "accepts_inbound": true, "secrets_key_fp": fp, "capabilities": caps, "hosts": hosts,
+ })
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ offlinePut(w.t, w.fx.rel, edgesFilePrefix+b.Box+".json", body)
+ sb := status.Box{Label: b.Box, GeneratedAt: time.Now()}
+ if !b.NoMetrics {
+ sb.Metrics = &status.BoxMetrics{HostClaimEnabled: b.ClaimsOn}
+ }
+ if err := status.WriteBox(context.Background(), w.fx.rel, sb); err != nil {
+ w.t.Fatal(err)
+ }
+ }
+}
+
+func (w *offlineWorld) req(method, path, body string) (int, string) {
+ w.t.Helper()
+ code, _, out := rawReq(w.t, method, w.base+path, w.token, body)
+ return code, out
+}
+
+// bucketState is every object in the bucket, for the "a refusal writes
+// nothing" check.
+func bucketState(t *testing.T, rel release.Store) map[string]string {
+ t.Helper()
+ keys, err := rel.List(context.Background(), "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := map[string]string{}
+ for _, k := range keys {
+ b, _, err := rel.Get(context.Background(), k)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out[k] = string(b)
+ }
+ return out
+}
+
+// refused posts and wants exactly this refusal, with nothing written.
+func (w *offlineWorld) refused(method, path, body string, wantCode int, wantMsg string) {
+ w.t.Helper()
+ before := bucketState(w.t, w.fx.rel)
+ code, out := w.req(method, path, body)
+ if code != wantCode || errorText(out) != wantMsg {
+ w.t.Fatalf("%s %s = %d %s\nwant %d %q", method, path, code, out, wantCode, wantMsg)
+ }
+ after := bucketState(w.t, w.fx.rel)
+ if len(before) != len(after) {
+ w.t.Fatalf("%s %s: a refusal changed the bucket: %d objects before, %d after", method, path, len(before), len(after))
+ }
+ for k, v := range before {
+ if after[k] != v {
+ w.t.Fatalf("%s %s: a refusal changed %s", method, path, k)
+ }
+ }
+}
+
+func (w *offlineWorld) view(code int, out string) offlineView {
+ w.t.Helper()
+ var v offlineView
+ if err := json.Unmarshal([]byte(out), &v); err != nil {
+ w.t.Fatalf("%v in %s", err, out)
+ }
+ _ = code
+ return v
+}
+
+// ok posts and wants a 202 whose view has the state.
+func (w *offlineWorld) ok(path, body, wantState string) offlineView {
+ w.t.Helper()
+ code, out := w.req(http.MethodPost, path, body)
+ if code != http.StatusAccepted {
+ w.t.Fatalf("POST %s = %d %s", path, code, out)
+ }
+ v := w.view(code, out)
+ if v.State != wantState {
+ w.t.Fatalf("POST %s: state %q, want %q: %s", path, v.State, wantState, out)
+ }
+ return v
+}
+
+func (w *offlineWorld) status() offlineView {
+ w.t.Helper()
+ code, out := w.req(http.MethodGet, "", "")
+ if code != http.StatusOK {
+ w.t.Fatalf("GET status = %d %s", code, out)
+ }
+ return w.view(code, out)
+}
+
+func (w *offlineWorld) record() *provision.Record {
+ w.t.Helper()
+ rec, err := provision.GetRecord(context.Background(), w.fx.rel, "bab", "titogo", offlineHost)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ return rec
+}
+
+// edit applies record transitions as a fixture, through the record writer.
+func (w *offlineWorld) edit(muts ...func(*provision.Record) error) {
+ w.t.Helper()
+ for _, m := range muts {
+ if _, err := provision.SetRecordMove(context.Background(), w.fx.rel, "bab", "titogo", offlineHost, m); err != nil {
+ w.t.Fatal(err)
+ }
+ }
+}
+
+func serialBytes(t *testing.T, rel release.Store) string {
+ t.Helper()
+ b, _, err := rel.Get(context.Background(), "control/serial")
+ if err != nil {
+ return ""
+ }
+ return string(b)
+}
+
+// start runs the start door and keeps the id.
+func (w *offlineWorld) start() offlineView {
+ w.t.Helper()
+ before := serialBytes(w.t, w.fx.rel)
+ v := w.ok("", `{"region":"uk"}`, provision.OfflineStopping)
+ if v.ID == "" {
+ w.t.Fatalf("start answered no id: %+v", v)
+ }
+ if serialBytes(w.t, w.fx.rel) == before {
+ w.t.Fatal("start did not bump the serial")
+ }
+ w.id = v.ID
+ return v
+}
+
+// report writes a box's report where the real box writes it, and returns
+// its bytes.
+func (w *offlineWorld) report(role, box, phase string, edit func(*moves.OfflineReport)) []byte {
+ w.t.Helper()
+ r := moves.OfflineReport{V: moves.OfflineReportVersion, Op: w.id, Host: offlineHost, Box: box, Role: role, Phase: phase,
+ At: time.Now().UTC(), Held: true, Fenced: true, Quiesced: true}
+ if edit != nil {
+ edit(&r)
+ }
+ body, err := json.Marshal(r)
+ if err != nil {
+ w.t.Fatal(err)
+ }
+ key := moves.OfflineReportKey(offlineHost, w.id, role, box)
+ if key == "" {
+ w.t.Fatalf("no report key for %s/%s", role, box)
+ }
+ offlinePut(w.t, w.fx.rel, key, body)
+ return body
+}
+
+// stopped writes every source box's report as a stopped box writes it.
+func (w *offlineWorld) stopped(edit func(box string, r *moves.OfflineReport), boxes ...string) {
+ w.t.Helper()
+ for _, b := range boxes {
+ w.report(moves.OfflineRoleSource, b, moves.OfflinePhaseStopped, func(r *moves.OfflineReport) {
+ if edit != nil {
+ edit(b, r)
+ }
+ })
+ }
+}
+
+func okChecks(owner string) map[string]moves.Check {
+ out := map[string]moves.Check{}
+ for _, it := range moves.OfflineOwned(owner) {
+ out[it.ID] = moves.Check{OK: true, Reason: "ok"}
+ }
+ return out
+}
+
+// sealerChecks is what a sealer's report carries when the seal went well:
+// every item the source side reports.
+func sealerChecks() map[string]moves.Check {
+ out := map[string]moves.Check{}
+ for _, it := range moves.OfflineSourced() {
+ out[it.ID] = moves.Check{OK: true, Reason: "ok"}
+ }
+ return out
+}
+
+// sealed plays the sealer: its report, and the transition that records
+// where it is.
+func (w *offlineWorld) sealed(sealer string) []byte {
+ w.t.Helper()
+ raw := w.report(moves.OfflineRoleSealer, sealer, moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
+ r.Items = sealerChecks()
+ r.TombstoneOwner = moves.TombstoneOwner(w.id)
+ r.TombstoneUntil = time.Now().Add(moves.TombstoneTTL)
+ })
+ sum := sha256.Sum256(raw)
+ key := moves.OfflineReportKey(offlineHost, w.id, moves.OfflineRoleSealer, sealer)
+ w.edit(provision.OfflineSealedBy(w.id, sealer, provision.MoveSource{FinalTXID: 7, Evidence: key + "@" + hex.EncodeToString(sum[:])}))
+ return raw
+}
+
+// landed plays the target: its landing report and the transition.
+func (w *offlineWorld) landed(target string) {
+ w.t.Helper()
+ w.report(moves.OfflineRoleTarget, target, moves.OfflinePhaseLanded, func(r *moves.OfflineReport) {
+ r.Items = okChecks(moves.OfflineOwnerLanding)
+ })
+ w.edit(provision.OfflineLandedBy(w.id, "uk"))
+}
+
+// toLanded drives a started move through seal and flip to offline-landed
+// with fresh reports from the one ie box and the one uk box.
+func (w *offlineWorld) toLanded() {
+ w.t.Helper()
+ w.start()
+ w.stopped(nil, "ie-box")
+ w.ok("/seal", "", provision.OfflineSealing)
+ w.sealed("ie-box")
+ w.stopped(nil, "ie-box")
+ w.ok("/flip", "", provision.OfflineLanding)
+ w.landed("uk-box")
+ w.stopped(nil, "ie-box")
+ w.sealerFresh("ie-box")
+}
+
+// sealerFresh rewrites the sealer's report as the sealer does every pass.
+func (w *offlineWorld) sealerFresh(sealer string) {
+ w.t.Helper()
+ w.report(moves.OfflineRoleSealer, sealer, moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
+ r.Items = sealerChecks()
+ r.TombstoneOwner = moves.TombstoneOwner(w.id)
+ r.TombstoneUntil = time.Now().Add(moves.TombstoneTTL)
+ })
+}
+
+var oneEach = []offlineBox{ieOff("ie-box"), ukOff("uk-box")}
+
+const (
+ msgO20 = "offline moves are not available while host claims or Phase 1 moves are on"
+ msgO29 = offlineHost + " has a host claim. offline moves are only for hosts without one"
+)
+
+func msgO21(box string) string {
+ return box + " cannot take part in an offline move. it runs an older release or has host claims on"
+}
+func msgO25(state string) string {
+ return "this move is " + state + ". run carlos move offline status --host " + offlineHost + " to see what it waits for"
+}
+func msgMC10(box string) string {
+ return "Waiting for " + box + " in ie to confirm it has stopped " + offlineHost + ". " + offlineHost + " is offline."
+}
+func msgO31(box string) string {
+ return box + " still runs a process for " + offlineHost + ". wait for it to stop, then flip"
+}
+
+var offlineRoutes = []struct{ method, path, body string }{
+ {http.MethodGet, "", ""},
+ {http.MethodPost, "", `{"region":"uk"}`},
+ {http.MethodPost, "/seal", ""},
+ {http.MethodPost, "/flip", ""},
+ {http.MethodPost, "/open", ""},
+ {http.MethodPost, "/finish", ""},
+ {http.MethodPost, "/abort", ""},
+}
+
+// Every door is the operator's: a member's token and an Instance-role
+// service token meet the same bytes http.NotFound writes, before MV-O20 or
+// anything else is said.
+func TestOfflineDoorsAreOperatorOnly(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ member := w.token
+ router := serviceToken(t, w.fx.st, OpsAccountID, "router", "titogo", svcrole.Instance)
+ if err := w.fx.st.SetOperator("alice@keymail.dev", false); err != nil {
+ t.Fatal(err)
+ }
+ want := canonicalNotFound(t)
+ for _, who := range []struct{ name, token string }{{"a member", member}, {"an instance token", router}} {
+ for _, rt := range offlineRoutes {
+ code, h, body := rawReq(t, rt.method, w.base+rt.path, who.token, rt.body)
+ assertNotFound(t, rt.method+" "+rt.path+" as "+who.name, want, code, h, body)
+ }
+ }
+ // The switch being on does not move the 404 behind MV-O20.
+ w.fx.srv.cfg.MovesEnabled = true
+ for _, rt := range offlineRoutes {
+ code, h, body := rawReq(t, rt.method, w.base+rt.path, member, rt.body)
+ assertNotFound(t, rt.method+" "+rt.path+" as a member with Phase 1 on", want, code, h, body)
+ }
+ if rec := w.record(); rec.Move != nil {
+ t.Fatalf("a refused request wrote a move: %+v", rec.Move)
+ }
+}
+
+// MV-O20, from the console's own configuration and from the deployment's
+// backfill plan; each leaves the bucket as it found it.
+func TestOfflineDoorsRefuseUnderClaims(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ each := func(label string) {
+ t.Helper()
+ for _, rt := range offlineRoutes {
+ w.refused(rt.method, rt.path, rt.body, http.StatusConflict, msgO20)
+ }
+ _ = label
+ }
+ w.fx.srv.cfg.HostClaim = true
+ each("claims, no authority")
+ w.fx.srv.cfg.ClaimAuthority = &hostops.ProductionAuthority{}
+ each("claims, with an authority")
+ w.fx.srv.cfg.HostClaim, w.fx.srv.cfg.ClaimAuthority = false, nil
+ w.fx.srv.cfg.MovesEnabled = true
+ each("Phase 1")
+ w.fx.srv.cfg.MovesEnabled = false
+
+ plan := func(phase string) {
+ t.Helper()
+ raw, err := release.SignRootEnvelope(release.RootSigner(w.fx.priv), hostops.BackfillPlan{V: 1, ID: "plan-1", Phase: phase, CreatedAt: time.Now().UTC()})
+ if err != nil {
+ t.Fatal(err)
+ }
+ offlinePut(t, w.fx.rel, hostops.BackfillKey, raw)
+ }
+ plan("collecting")
+ w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO20)
+ plan("adopting")
+ w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO20)
+ offlinePut(t, w.fx.rel, hostops.BackfillKey, []byte("not an envelope"))
+ w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, msgO20)
+ // A plan at done is allowed, with any number of hosts.
+ plan("done")
+ w.start()
+}
+
+func TestOfflineStartRefusals(t *testing.T) {
+ type tc struct {
+ name string
+ boxes []offlineBox
+ setup func(w *offlineWorld)
+ body string
+ code int
+ msg string
+ }
+ const body = `{"region":"uk"}`
+ other := func(w *offlineWorld) {
+ // A second instance, moving offline.
+ u := instancesURL(w.fx, "bab", "titogo")
+ if c, _, out := rawReq(t, http.MethodPost, u, w.token, `{"host":"beta.go.tito.io","region":"ie"}`); c != http.StatusCreated {
+ t.Fatalf("create = %d %s", c, out)
+ }
+ if _, err := provision.SetRecordMove(context.Background(), w.fx.rel, "bab", "titogo", "beta.go.tito.io",
+ provision.OfflineStart("mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", "ie", "uk", "x", time.Now())); err != nil {
+ t.Fatal(err)
+ }
+ }
+ cases := []tc{
+ {name: "a replica", boxes: oneEach, body: body, code: 400, msg: "a replica does not move. delete it and create one in the new region",
+ setup: func(w *offlineWorld) { w.edit(func(r *provision.Record) error { r.Role = "replica"; return nil }) }},
+ {name: "replicas", boxes: oneEach, body: body, code: 400, msg: "a home with replicas does not move. remove its replicas first",
+ setup: func(w *offlineWorld) {
+ w.edit(func(r *provision.Record) error { r.Replicas = []string{"uk"}; return nil })
+ }},
+ {name: "steering", boxes: oneEach, body: body, code: 400, msg: offlineHost + " is steered or runs under a unit. offline moves cannot carry either yet",
+ setup: func(w *offlineWorld) { w.edit(func(r *provision.Record) error { r.Steering = "geo"; return nil }) }},
+ {name: "a unit", boxes: oneEach, body: body, code: 400, msg: offlineHost + " is steered or runs under a unit. offline moves cannot carry either yet",
+ setup: func(w *offlineWorld) { w.edit(func(r *provision.Record) error { r.Unit = "app.service"; return nil }) }},
+ {name: "a record acquisition", boxes: oneEach, body: body, code: 409, msg: msgO29,
+ setup: func(w *offlineWorld) { w.edit(func(r *provision.Record) error { r.Acquisition = "acq-1"; return nil }) }},
+ {name: "a host claim", boxes: oneEach, body: body, code: 409, msg: msgO29,
+ setup: func(w *offlineWorld) { claimFor(t, w.fx, offlineHost, "ie-box", "ie") }},
+ {name: "a domains claim with an acquisition", boxes: oneEach, body: body, code: 409, msg: msgO29,
+ setup: func(w *offlineWorld) {
+ c := domains.Claim{Schema: 1, Host: offlineHost, Account: "bab", App: "titogo", Acquisition: "acq-1"}
+ raw, _ := json.Marshal(c)
+ offlinePut(t, w.fx.rel, domains.ClaimKey(offlineHost), raw)
+ }},
+ {name: "an admission", boxes: oneEach, body: body, code: 409, msg: msgO29,
+ setup: func(w *offlineWorld) {
+ key, err := hostops.AdmissionKey("bab", "titogo")
+ if err != nil {
+ t.Fatal(err)
+ }
+ offlinePut(t, w.fx.rel, key, []byte(`{}`))
+ }},
+ {name: "a second move in flight", boxes: oneEach, body: body, code: 409, msg: "beta.go.tito.io is moving offline. move one instance at a time", setup: other},
+ {name: "no region", boxes: oneEach, body: `{"region":""}`, code: 400, msg: "invalid region"},
+ {name: "a bad region", boxes: oneEach, body: `{"region":"UK!"}`, code: 400, msg: "invalid region"},
+ {name: "the current region", boxes: oneEach, body: `{"region":"ie"}`, code: 409, msg: "the instance is already in ie"},
+ {name: "the manifest's regions", boxes: oneEach, body: body, code: 400, msg: "region uk is not one of this app's regions: ie",
+ setup: func(w *offlineWorld) {
+ m := titoManifest()
+ m.Pool = ""
+ enableInstances(t, w.fx, "titogo", m)
+ }},
+ {name: "an unreadable edges file", boxes: oneEach, body: body, code: 409, msg: "the fleet's edges files could not all be read. try again",
+ setup: func(w *offlineWorld) { offlinePut(t, w.fx.rel, edgesFilePrefix+"broken.json", []byte("{")) }},
+ {name: "no source box", boxes: []offlineBox{ukOff("uk-box")}, body: body, code: 400, msg: "no eligible box in ie publishes an edges file"},
+ {name: "no target box", boxes: []offlineBox{ieOff("ie-box")}, body: body, code: 409, msg: "uk has 0 boxes. an offline move needs exactly one box in the region it moves to"},
+ {name: "two target boxes", boxes: []offlineBox{ieOff("ie-box"), ukOff("uk-a"), ukOff("uk-b")}, body: body, code: 409,
+ msg: "uk has 2 boxes. an offline move needs exactly one box in the region it moves to"},
+ {name: "a source box without the capability", boxes: []offlineBox{func() offlineBox { b := ieOff("ie-box"); b.NoCap = true; return b }(), ukOff("uk-box")},
+ body: body, code: 409, msg: msgO21("ie-box")},
+ {name: "the second source box without the capability", boxes: []offlineBox{ieOff("ie-a"), func() offlineBox { b := ieOff("ie-b"); b.NoCap = true; return b }(), ukOff("uk-box")},
+ body: body, code: 409, msg: msgO21("ie-b")},
+ {name: "a target box without the capability", boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.NoCap = true; return b }()},
+ body: body, code: 409, msg: msgO21("uk-box")},
+ {name: "a target that does not hold the secrets key", boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.FP = "not-it"; return b }()},
+ body: body, code: 409, msg: "uk-box does not hold this deployment's secrets key"},
+ {name: "a target that lists the host", boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.Hosts = []string{offlineHost}; return b }()},
+ body: body, code: 409, msg: "uk-box already serves " + offlineHost + ". an offline move needs the region it moves to to be empty of it"},
+ {name: "a target with no data bucket", boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.Bucket = ""; return b }()},
+ body: body, code: 400, msg: "region uk cannot take a move in this deployment. it publishes no staging bucket. Regions that can: ie"},
+ // Rule 3: a claims-on box in either region, judged from its published state.
+ {name: "a claims-on box in the source region that serves the shared pool", body: body, code: 409, msg: msgO21("ie-claims"),
+ boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ieOff("ie-claims"); b.ClaimsOn = true; b.NoCap = true; return b }(), ukOff("uk-box")}},
+ {name: "a claims-on box in the target region whose pools do not match (no census evidence is published)", body: body, code: 409, msg: msgO21("uk-claims"),
+ boxes: []offlineBox{ieOff("ie-box"), ukOff("uk-box"), func() offlineBox {
+ b := ukOff("uk-claims")
+ b.ClaimsOn = true
+ b.NoCap = true
+ b.NotSharedPool = true
+ return b
+ }()}},
+ {name: "a claims-on box that publishes a row for the host", body: body, code: 409, msg: msgO21("ie-claims"),
+ boxes: []offlineBox{ieOff("ie-box"), func() offlineBox {
+ b := ieOff("ie-claims")
+ b.ClaimsOn, b.NoCap, b.NotSharedPool, b.Hosts = true, true, true, []string{offlineHost}
+ return b
+ }(), ukOff("uk-box")}},
+ {name: "a box whose report says claims are on although it carries the capability", body: body, code: 409, msg: msgO21("uk-box"),
+ boxes: []offlineBox{ieOff("ie-box"), func() offlineBox { b := ukOff("uk-box"); b.ClaimsOn = true; return b }()}},
+ {name: "a box outside both regions that runs claims is not consulted", body: "", code: 202,
+ boxes: []offlineBox{ieOff("ie-box"), ukOff("uk-box"), func() offlineBox {
+ b := offlineBox{Box: "se-box", Placement: "se", Bucket: "data-se"}
+ b.ClaimsOn, b.NoCap = true, true
+ return b
+ }()}},
+ {name: "a box with no metrics and no capability in the region cannot be judged", body: body, code: 409, msg: msgO21("ie-old"),
+ boxes: []offlineBox{ieOff("ie-box"), func() offlineBox {
+ b := ieOff("ie-old")
+ b.NoMetrics, b.NoCap, b.NotSharedPool = true, true, true
+ return b
+ }(), ukOff("uk-box")}},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ w := newOfflineWorld(t, c.boxes...)
+ if c.setup != nil {
+ c.setup(w)
+ }
+ if c.code == 202 {
+ w.start()
+ return
+ }
+ w.refused(http.MethodPost, "", c.body, c.code, c.msg)
+ if rec := w.record(); rec.Move != nil && c.name != "a second move in flight" {
+ t.Fatalf("a refused start left a move: %+v", rec.Move)
+ }
+ })
+ }
+}
+
+// A done backfill plan does not refuse, and the claims-on truth table is
+// all three conditions: one missing is a refusal.
+func TestOfflineClaimsOnBoxNeedsAllThree(t *testing.T) {
+ yes, no := true, false
+ cases := []struct {
+ name string
+ box offlineClaimsOnBox
+ ok bool
+ }{
+ {"all three", offlineClaimsOnBox{Census: &yes, SharedPool: &no}, true},
+ {"a row for the host", offlineClaimsOnBox{HostRow: true, Census: &yes, SharedPool: &no}, false},
+ {"census not present", offlineClaimsOnBox{Census: &no, SharedPool: &no}, false},
+ {"census not said", offlineClaimsOnBox{SharedPool: &no}, false},
+ {"pools match", offlineClaimsOnBox{Census: &yes, SharedPool: &yes}, false},
+ {"pools not said", offlineClaimsOnBox{Census: &yes}, false},
+ }
+ for _, c := range cases {
+ if got := c.box.ok(); got != c.ok {
+ t.Errorf("%s: ok = %v, want %v", c.name, got, c.ok)
+ }
+ }
+ // No report field carries the census today, so no claims-on box passes.
+ if offlineCensusReported(&status.BoxMetrics{HostClaimEnabled: true}) != nil {
+ t.Error("a box report says something about census.json this build does not know how to read")
+ }
+}
+
+func TestOfflineStartWritesTheMoveAndBumpsTheSerial(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ if v := w.status(); v.State != "none" || v.Host != offlineHost {
+ t.Fatalf("status before any move = %+v", v)
+ }
+ v := w.start()
+ if v.From != "ie" || v.To != "uk" || v.Host != offlineHost || !v.CanAbort || v.RequestedBy != "alice@keymail.dev" {
+ t.Fatalf("start view = %+v", v)
+ }
+ rec := w.record()
+ if rec.Move == nil || rec.Move.Kind != provision.MoveKindOffline || rec.Move.State != provision.OfflineStopping ||
+ rec.Move.From != "ie" || rec.Move.To != "uk" || rec.Move.ID != v.ID || rec.Region != "ie" {
+ t.Fatalf("record after start = %+v region %q", rec.Move, rec.Region)
+ }
+ // Waiting for the boxes, in the move's own words; no verb is next.
+ if v.Next != "" || v.Waiting != msgMC10("ie-box") {
+ t.Fatalf("view waits for %q (next %q)", v.Waiting, v.Next)
+ }
+ // A second start is refused by the record.
+ w.refused(http.MethodPost, "", `{"region":"uk"}`, http.StatusConflict, "a move to uk is already offline-stopping")
+ // The status now carries the manifest with the door-owned class row ok.
+ got := w.status()
+ if got.State != provision.OfflineStopping || len(got.Items) != len(moves.OfflineManifest) {
+ t.Fatalf("status = %+v", got)
+ }
+ for _, it := range got.Items {
+ switch it.Owner {
+ case moves.OfflineOwnerDoor:
+ if it.Status != "ok" {
+ t.Errorf("%s: %s", it.ID, it.Status)
+ }
+ case moves.OfflineOwnerNone:
+ if it.Status != "unchecked" {
+ t.Errorf("%s: %s", it.ID, it.Status)
+ }
+ default:
+ if it.Status != "waiting" {
+ t.Errorf("%s: %s", it.ID, it.Status)
+ }
+ }
+ }
+}
+
+func TestOfflineSealDoor(t *testing.T) {
+ w := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box"))
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, offlineHost+" has no offline move in flight")
+ w.start()
+ // No report yet: the first box is named.
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgMC10("ie-a"))
+ // A box that has not held, fenced and quiesced is named.
+ for _, lost := range []string{"held", "fenced", "quiesced"} {
+ w.stopped(func(box string, r *moves.OfflineReport) {
+ if box == "ie-b" {
+ switch lost {
+ case "held":
+ r.Held = false
+ case "fenced":
+ r.Fenced = false
+ case "quiesced":
+ r.Quiesced = false
+ }
+ }
+ }, "ie-a", "ie-b")
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgMC10("ie-b"))
+ }
+ // A stale report is not a report.
+ w.stopped(func(box string, r *moves.OfflineReport) {
+ if box == "ie-b" {
+ r.At = time.Now().Add(-5 * time.Minute)
+ }
+ }, "ie-a", "ie-b")
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgMC10("ie-b"))
+ // A box still listing the host has not stopped serving it.
+ w.stopped(nil, "ie-a", "ie-b")
+ still := ieOff("ie-b")
+ still.Hosts = []string{offlineHost}
+ w.fleet(still)
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgMC10("ie-b"))
+ w.fleet(ieOff("ie-b"))
+ // A sidecar anywhere.
+ w.stopped(func(box string, r *moves.OfflineReport) { r.Sidecar = box == "ie-b" }, "ie-a", "ie-b")
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, "ie-b keeps a sidecar file for "+offlineHost+". instances with sidecars cannot move yet")
+ // Local data on two boxes.
+ w.stopped(func(box string, r *moves.OfflineReport) { r.LocalDB = true; r.Files = box == "ie-b" }, "ie-a", "ie-b")
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, offlineHost+" keeps local data on more than one box: ie-a, ie-b. remove the stale copy first")
+ // One box with data is the sealer, even when it is not the first.
+ before := serialBytes(t, w.fx.rel)
+ w.stopped(func(box string, r *moves.OfflineReport) { r.Env = box == "ie-b" }, "ie-a", "ie-b")
+ v := w.ok("/seal", "", provision.OfflineSealing)
+ if v.Sealer != "ie-b" || strings.Join(v.Boxes, ",") != "ie-a,ie-b" {
+ t.Fatalf("sealer %q boxes %v", v.Sealer, v.Boxes)
+ }
+ if serialBytes(t, w.fx.rel) == before {
+ t.Fatal("seal did not bump the serial")
+ }
+ rec := w.record()
+ if rec.Move.State != provision.OfflineSealing || rec.Move.Sealer != "ie-b" || len(rec.Move.Boxes) != 2 {
+ t.Fatalf("record = %+v", rec.Move)
+ }
+ // seal is the verb of stopping only.
+ w.refused(http.MethodPost, "/seal", "", http.StatusConflict, msgO25(provision.OfflineSealing))
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealing))
+}
+
+func TestOfflineSealWithNoDataTakesTheFirstBox(t *testing.T) {
+ w := newOfflineWorld(t, ieOff("ie-a"), ieOff("ie-b"), ukOff("uk-box"))
+ w.start()
+ w.stopped(func(box string, r *moves.OfflineReport) { r.RowPresent = false }, "ie-a", "ie-b")
+ if v := w.ok("/seal", "", provision.OfflineSealing); v.Sealer != "ie-a" {
+ t.Fatalf("sealer = %q", v.Sealer)
+ }
+}
+
+func TestOfflineFlipDoor(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.start()
+ w.stopped(nil, "ie-box")
+ w.ok("/seal", "", provision.OfflineSealing)
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealing))
+ // The sealer's report is not what the record names.
+ w.sealed("ie-box")
+ w.stopped(nil, "ie-box")
+ w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
+ r.Items = sealerChecks()
+ r.FinalTXID = 99 // a different report from the one the record hashed
+ })
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
+ // A seal item that is not OK.
+ raw := w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
+ r.Items = sealerChecks()
+ r.Items["files"] = moves.Check{OK: false, Reason: "a file the archive refuses"}
+ })
+ w.rehash("ie-box", raw)
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO25(provision.OfflineSealed))
+ good := w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) { r.Items = sealerChecks() })
+ w.rehash("ie-box", good)
+ // A process left, a layout journal, a stale report.
+ w.stopped(func(_ string, r *moves.OfflineReport) { r.Procs = 1 }, "ie-box")
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box"))
+ w.stopped(func(_ string, r *moves.OfflineReport) { r.LayoutJournal = true }, "ie-box")
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box"))
+ w.stopped(func(_ string, r *moves.OfflineReport) { r.At = time.Now().Add(-3 * time.Minute) }, "ie-box")
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgO31("ie-box"))
+ w.stopped(func(_ string, r *moves.OfflineReport) { r.Fenced = false }, "ie-box")
+ w.refused(http.MethodPost, "/flip", "", http.StatusConflict, msgMC10("ie-box"))
+ // All clear.
+ w.stopped(nil, "ie-box")
+ before := serialBytes(t, w.fx.rel)
+ v := w.ok("/flip", "", provision.OfflineLanding)
+ if v.Next != "" {
+ t.Fatalf("next after flip = %q", v.Next)
+ }
+ if serialBytes(t, w.fx.rel) == before {
+ t.Fatal("flip did not bump the serial")
+ }
+ if rec := w.record(); rec.Region != "uk" || rec.Move.State != provision.OfflineLanding {
+ t.Fatalf("after flip: region %q move %+v", rec.Region, rec.Move)
+ }
+}
+
+// rehash points the record's source evidence at a report's bytes, as the
+// sealer's transition does when it records its own report.
+func (w *offlineWorld) rehash(sealer string, raw []byte) {
+ w.t.Helper()
+ sum := sha256.Sum256(raw)
+ key := moves.OfflineReportKey(offlineHost, w.id, moves.OfflineRoleSealer, sealer)
+ w.edit(func(r *provision.Record) error {
+ r.Move.Source.Evidence = key + "@" + hex.EncodeToString(sum[:])
+ return nil
+ })
+}
+
+func TestOfflineOpenDoorRechecksTheStop(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.toLanded()
+ // The listing check of a source box is not the door's: the door reads its report.
+ reset := func() {
+ w.stopped(nil, "ie-box")
+ w.sealerFresh("ie-box")
+ w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseLanded, func(r *moves.OfflineReport) { r.Items = okChecks(moves.OfflineOwnerLanding) })
+ }
+ reset()
+ // A stale sealer report.
+ w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
+ r.Items = sealerChecks()
+ r.At = time.Now().Add(-3 * time.Minute)
+ r.TombstoneOwner = moves.TombstoneOwner(w.id)
+ r.TombstoneUntil = time.Now().Add(time.Hour)
+ })
+ w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO25(provision.OfflineLanded))
+ reset()
+ // A tombstone that is another move's, or none, or run out.
+ for name, edit := range map[string]func(*moves.OfflineReport){
+ "none": func(r *moves.OfflineReport) { r.TombstoneOwner = "" },
+ "another": func(r *moves.OfflineReport) { r.TombstoneOwner = moves.TombstoneOwner("mv_other") },
+ "expired": func(r *moves.OfflineReport) { r.TombstoneUntil = time.Now().Add(-time.Minute) },
+ "unstated": func(r *moves.OfflineReport) { r.TombstoneUntil = time.Time{} },
+ } {
+ w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseSealed, func(r *moves.OfflineReport) {
+ r.Items = sealerChecks()
+ r.TombstoneOwner = moves.TombstoneOwner(w.id)
+ r.TombstoneUntil = time.Now().Add(time.Hour)
+ edit(r)
+ })
+ code, out := w.req(http.MethodPost, "/open", "")
+ if code != http.StatusConflict || errorText(out) != msgO25(provision.OfflineLanded) {
+ t.Fatalf("tombstone %s: %d %s", name, code, out)
+ }
+ reset()
+ }
+ // A source report that lost Fenced, or went stale.
+ w.stopped(func(_ string, r *moves.OfflineReport) { r.Fenced = false }, "ie-box")
+ w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgMC10("ie-box"))
+ w.stopped(func(_ string, r *moves.OfflineReport) { r.At = time.Now().Add(-10 * time.Minute) }, "ie-box")
+ w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgMC10("ie-box"))
+ reset()
+ // A landing item that is not OK.
+ w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseLanded, func(r *moves.OfflineReport) {
+ r.Items = okChecks(moves.OfflineOwnerLanding)
+ r.Items["config"] = moves.Check{OK: false, Reason: "the render waits"}
+ })
+ w.refused(http.MethodPost, "/open", "", http.StatusConflict, msgO25(provision.OfflineLanded))
+ // The status view says the same, and names no verb.
+ if v := w.status(); v.Next != "" || v.Waiting != msgO25(provision.OfflineLanded) {
+ t.Fatalf("status = next %q waiting %q", v.Next, v.Waiting)
+ } else {
+ for _, it := range v.Items {
+ if it.ID == "config" && (it.Status != "waiting" || it.Detail != "the render waits") {
+ t.Fatalf("config item = %+v", it)
+ }
+ }
+ }
+ reset()
+ if v := w.status(); v.Next != offlineVerbOpen {
+ t.Fatalf("status next = %q (%s)", v.Next, v.Waiting)
+ }
+ before := serialBytes(t, w.fx.rel)
+ w.ok("/open", "", provision.OfflineOpen)
+ if serialBytes(t, w.fx.rel) == before {
+ t.Fatal("open did not bump the serial")
+ }
+ if rec := w.record(); rec.Move.State != provision.OfflineOpen || rec.Region != "uk" || rec.Move.OpenedAt == nil {
+ t.Fatalf("after open: %+v region %q", rec.Move, rec.Region)
+ }
+}
+
+func TestOfflineFinishDoor(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.toLanded()
+ w.ok("/open", "", provision.OfflineOpen)
+ // open to finishing: the target's opened and every source's parked.
+ w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineOpen))
+ w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseOpened, func(r *moves.OfflineReport) { r.Items = okChecks(moves.OfflineOwnerLanding) })
+ w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineOpen))
+ w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseParked, nil)
+ before := serialBytes(t, w.fx.rel)
+ w.ok("/finish", "", provision.OfflineFinishing)
+ if serialBytes(t, w.fx.rel) == before {
+ t.Fatal("finish did not bump the serial")
+ }
+ // finishing to cleared: the sealer lifted and every source gone.
+ w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineFinishing))
+ w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseLifted, nil)
+ w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineFinishing))
+ w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseGone, nil)
+ v := w.ok("/finish", "", "none")
+ if v.Last == nil || v.Last.Outcome != provision.OfflineOutcomeDone || v.Last.ID != w.id {
+ t.Fatalf("last = %+v", v.Last)
+ }
+ if rec := w.record(); rec.Move != nil || rec.Region != "uk" || rec.LastMove == nil {
+ t.Fatalf("after finish: move %+v region %q", rec.Move, rec.Region)
+ }
+ w.refused(http.MethodPost, "/finish", "", http.StatusConflict, offlineHost+" has no offline move in flight")
+ // The status of a host with no move is the outcome of the last.
+ if s := w.status(); s.State != "none" || s.Last == nil {
+ t.Fatalf("status = %+v", s)
+ }
+}
+
+func TestOfflineAbortDoor(t *testing.T) {
+ // Before the flip: the region is untouched.
+ w := newOfflineWorld(t, oneEach...)
+ w.refused(http.MethodPost, "/abort", "", http.StatusConflict, offlineHost+" has no offline move in flight")
+ w.start()
+ before := serialBytes(t, w.fx.rel)
+ w.ok("/abort", "", provision.OfflineReverting)
+ if serialBytes(t, w.fx.rel) == before {
+ t.Fatal("abort did not bump the serial")
+ }
+ if rec := w.record(); rec.Region != "ie" || rec.Move.State != provision.OfflineReverting {
+ t.Fatalf("after abort: %q %+v", rec.Region, rec.Move)
+ }
+ // A second abort writes nothing.
+ state := bucketState(t, w.fx.rel)
+ w.ok("/abort", "", provision.OfflineReverting)
+ for k, v := range bucketState(t, w.fx.rel) {
+ if state[k] != v {
+ t.Fatalf("a second abort changed %s", k)
+ }
+ }
+ // reverting to cleared: every source box's reverted. Aborted before the
+ // seal there is no sealer to ask, and no target either.
+ w.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting))
+ w.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil)
+ if v := w.ok("/finish", "", "none"); v.Last == nil || v.Last.Outcome != provision.OfflineOutcomeAborted {
+ t.Fatalf("last = %+v", v.Last)
+ }
+
+ // After the flip: the record goes back to the source.
+ w2 := newOfflineWorld(t, oneEach...)
+ w2.start()
+ w2.stopped(nil, "ie-box")
+ w2.ok("/seal", "", provision.OfflineSealing)
+ w2.sealed("ie-box")
+ w2.stopped(nil, "ie-box")
+ w2.ok("/flip", "", provision.OfflineLanding)
+ if rec := w2.record(); rec.Region != "uk" {
+ t.Fatalf("region after flip = %q", rec.Region)
+ }
+ w2.ok("/abort", "", provision.OfflineReverting)
+ if rec := w2.record(); rec.Region != "ie" {
+ t.Fatalf("region after abort = %q", rec.Region)
+ }
+ // Sealed and then aborted: the sealer's reverted is asked for too, and
+ // the target's, which may be a box that died, is not.
+ w2.report(moves.OfflineRoleSource, "ie-box", moves.OfflinePhaseReverted, nil)
+ w2.refused(http.MethodPost, "/finish", "", http.StatusConflict, msgO25(provision.OfflineReverting))
+ w2.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseReverted, nil)
+ if v := w2.ok("/finish", "", "none"); v.Last == nil || v.Last.Outcome != provision.OfflineOutcomeAborted {
+ t.Fatalf("last = %+v", v.Last)
+ }
+ if rec := w2.record(); rec.Region != "ie" || rec.Move != nil {
+ t.Fatalf("after finish: %q %+v", rec.Region, rec.Move)
+ }
+
+ // From open on, abort is MV-O27 and writes nothing.
+ w3 := newOfflineWorld(t, oneEach...)
+ w3.toLanded()
+ w3.ok("/open", "", provision.OfflineOpen)
+ w3.refused(http.MethodPost, "/abort", "", http.StatusConflict, offlineHost+" is already served from uk. move it back with a new move")
+ w3.edit(provision.OfflineFinishingTransition(w3.id))
+ w3.refused(http.MethodPost, "/abort", "", http.StatusConflict, offlineHost+" is already served from uk. move it back with a new move")
+}
+
+// The status view is the move, every report, the manifest with each item's
+// latest check and the next legal verb.
+func TestOfflineStatusView(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.toLanded()
+ v := w.status()
+ if v.State != provision.OfflineLanded || v.ID != w.id || v.Sealer != "ie-box" || v.From != "ie" || v.To != "uk" || !v.CanAbort || v.Next != offlineVerbOpen {
+ t.Fatalf("status = %+v", v)
+ }
+ roles := map[string]bool{}
+ for _, r := range v.Reports {
+ roles[r.Role+"/"+r.Box] = true
+ if !r.Fresh {
+ t.Errorf("report %s/%s is not fresh", r.Role, r.Box)
+ }
+ }
+ for _, want := range []string{"source/ie-box", "sealer/ie-box", "target/uk-box"} {
+ if !roles[want] {
+ t.Errorf("no %s report in the view: %v", want, roles)
+ }
+ }
+ if len(v.Items) != len(moves.OfflineManifest) {
+ t.Fatalf("%d items, want %d", len(v.Items), len(moves.OfflineManifest))
+ }
+ for _, it := range v.Items {
+ if (it.Owner == moves.OfflineOwnerSeal || it.Owner == moves.OfflineOwnerLanding) && it.Status != "ok" {
+ t.Errorf("item %s is %s", it.ID, it.Status)
+ }
+ }
+ // The step that owns an item decides it: a landing item is the target's
+ // to prove, a seal item the sealer's. A failed phase marks a non-OK
+ // item failed; a landing phase marks it waiting.
+ w.report(moves.OfflineRoleSealer, "ie-box", moves.OfflinePhaseFailed, func(r *moves.OfflineReport) {
+ r.Items = sealerChecks()
+ r.Items["db-tail"] = moves.Check{OK: false, Reason: "the replica is behind"}
+ r.Items["files"] = moves.Check{OK: false, Reason: "the sealer's own view does not decide it"}
+ })
+ w.report(moves.OfflineRoleTarget, "uk-box", moves.OfflinePhaseLanded, func(r *moves.OfflineReport) {
+ r.Items = okChecks(moves.OfflineOwnerLanding)
+ delete(r.Items, "release")
+ r.Items["files"] = moves.Check{OK: false, Reason: "the digest differs"}
+ })
+ seen := map[string]offlineItemView{}
+ for _, it := range w.status().Items {
+ seen[it.ID] = it
+ }
+ if it := seen["db-tail"]; it.Status != "failed" || it.Detail != "the replica is behind" {
+ t.Errorf("db-tail = %+v", it)
+ }
+ if it := seen["files"]; it.Status != "waiting" || it.Detail != "the digest differs" || it.Owner != moves.OfflineOwnerLanding || it.Source != moves.OfflineSourceSeal {
+ t.Errorf("files = %+v", it)
+ }
+ if it := seen["release"]; it.Status != "waiting" {
+ t.Errorf("release = %+v", it)
+ }
+ // A report that is not this move's, or not at its own key, is not in the view.
+ other := moves.OfflineReport{V: 1, Op: "mv_other", Host: offlineHost, Box: "ie-box", Role: moves.OfflineRoleSource, Phase: moves.OfflinePhaseStopped, At: time.Now()}
+ body, _ := json.Marshal(other)
+ offlinePut(t, w.fx.rel, moves.OfflineReportKey(offlineHost, w.id, moves.OfflineRoleSource, "ie-x"), body)
+ for _, r := range w.status().Reports {
+ if r.Box == "ie-x" {
+ t.Errorf("a report at the wrong key is in the view: %+v", r)
+ }
+ }
+}
+
+// Evidence matching takes the report's hash bare or after its key.
+func TestOfflineEvidenceMatches(t *testing.T) {
+ raw := []byte(`{"x":1}`)
+ sum := sha256.Sum256(raw)
+ h := hex.EncodeToString(sum[:])
+ key := moves.OfflineReportKey(offlineHost, "mv_00000000000000000000000000", moves.OfflineRoleSealer, "ie-box")
+ for _, ok := range []string{h, "sha256:" + h, key + "@" + h, key + ":" + h, key + " " + h, key + "#sha256:" + h} {
+ if !offlineEvidenceMatches(ok, key, raw) {
+ t.Errorf("%q should match", ok)
+ }
+ }
+ for _, bad := range []string{"", key, key + "@", key + "@" + strings.Repeat("0", 64), "other/key@" + h, h + "0", key + h} {
+ if offlineEvidenceMatches(bad, key, raw) {
+ t.Errorf("%q should not match", bad)
+ }
+ }
+}
+
+// A transition's refusal reaches the operator as a recorded sentence and
+// never as the error's own text: the offline transitions' rows pass as they
+// are (the values in them are the record's own), and anything else is
+// MV-N46.
+func TestOfflineConflictMapsOntoRecordedSentences(t *testing.T) {
+ w := newOfflineWorld(t, oneEach...)
+ w.start()
+ apply := func(mut func(*provision.Record) error) error {
+ _, err := provision.SetRecordMove(context.Background(), w.fx.rel, "bab", "titogo", offlineHost, mut)
+ return err
+ }
+ const other = "mv_ZZZZZZZZZZZZZZZZZZZZZZZZZZ"
+ cases := []struct {
+ name string
+ mut func(*provision.Record) error
+ want string
+ }{
+ {"wrong state", provision.OfflineFlip(w.id), "this offline move is offline-stopping. this step follows offline-sealed"},
+ {"another move's id", provision.OfflineSeal(other, "ie-box", []string{"ie-box"}), "the record carries a different offline move"},
+ {"a seal without its sealer among the boxes", provision.OfflineSeal(w.id, "ie-x", []string{"ie-box"}),
+ "a seal names the box that seals and the boxes that stopped the instance. the box that seals is one of them"},
+ {"an abort already past", provision.OfflineFinish(w.id, time.Now()), "this offline move is offline-stopping. it finishes from finishing or reverting"},
+ }
+ for _, c := range cases {
+ err := apply(c.mut)
+ if err == nil {
+ t.Fatalf("%s: accepted", c.name)
+ }
+ if got := offlineConflict(err); got != c.want {
+ t.Errorf("%s: %q, want %q", c.name, got, c.want)
+ }
+ }
+ // A refusal this table does not know is the fixed sentence.
+ if got := offlineConflict(fmt.Errorf("%w: something new and unreviewed", provision.ErrMoveConflict)); got != "this move's state does not allow that. read its state and try again" {
+ t.Errorf("unknown refusal = %q", got)
+ }
+ // Once open, an abort is refused with its recorded sentence.
+ w.stopped(nil, "ie-box")
+ w.ok("/seal", "", provision.OfflineSealing)
+ w.sealed("ie-box")
+ w.stopped(nil, "ie-box")
+ w.ok("/flip", "", provision.OfflineLanding)
+ w.landed("uk-box")
+ w.edit(provision.OfflineOpenTransition(w.id, time.Now()))
+ err := apply(provision.OfflineAbort(w.id))
+ if got := offlineConflict(err); got != "this offline move is offline-open. it cannot be aborted once it is open" {
+ t.Errorf("abort after open = %q", got)
+ }
+ // A legacy move meets "this is not an offline move".
+ if got := offlineConflict(fmt.Errorf("%w: %s", provision.ErrMoveConflict, "this is not an offline move")); got != "this is not an offline move" {
+ t.Errorf("not offline = %q", got)
+ }
+}
+
+// The flagship's shape: the home placement is the record's empty region and
+// the console names it. A move out of home reads the placementless boxes, a
+// move into it the same, and the operator reads the home by its name.
+func TestOfflineMovesOutOfAndIntoHome(t *testing.T) {
+ homeBox := offlineBox{Box: "home-box", Bucket: "data-ie"}
+ w := newOfflineWorldIn(t, "ie", "ie", homeBox, ukOff("uk-box"))
+ if rec := w.record(); rec.Region != "" {
+ t.Fatalf("the record's region = %q, want the home's empty code", rec.Region)
+ }
+ w.refused(http.MethodPost, "", `{"region":"ie"}`, http.StatusConflict, "the instance is already in ie")
+ v := w.start()
+ if v.From != "ie" || v.To != "uk" || v.Waiting != "Waiting for home-box in ie to confirm it has stopped "+offlineHost+". "+offlineHost+" is offline." {
+ t.Fatalf("view = %+v", v)
+ }
+ if rec := w.record(); rec.Move.From != "" || rec.Move.To != "uk" || rec.Region != "" {
+ t.Fatalf("record = %+v region %q", rec.Move, rec.Region)
+ }
+ w.report(moves.OfflineRoleSource, "home-box", moves.OfflinePhaseStopped, nil)
+ if v := w.ok("/seal", "", provision.OfflineSealing); v.Sealer != "home-box" {
+ t.Fatalf("sealer = %q", v.Sealer)
+ }
+
+ // Back the other way: from uk into home.
+ b := newOfflineWorldIn(t, "ie", "uk", homeBox, ukOff("uk-box"))
+ bv := b.ok("", `{"region":"ie"}`, provision.OfflineStopping)
+ if bv.From != "uk" || bv.To != "ie" {
+ t.Fatalf("view = %+v", bv)
+ }
+ if rec := b.record(); rec.Move.From != "uk" || rec.Move.To != "" || rec.Region != "uk" {
+ t.Fatalf("record = %+v region %q", rec.Move, rec.Region)
+ }
+
+ // A box that sets CARLOS_REGION to the home's name is home to the console
+ // and not to itself: it cannot take a move into home.
+ named := offlineBox{Box: "ie-named", Placement: "ie", Bucket: "data-ie"}
+ c := newOfflineWorldIn(t, "ie", "uk", homeBox, ukOff("uk-box"), named)
+ c.refused(http.MethodPost, "", `{"region":"ie"}`, http.StatusConflict, "ie-named sets CARLOS_REGION, so it cannot take a move into the home region")
+}
diff --git a/internal/console/authcli.go b/internal/console/authcli.go
index 65d9e32a1..8733e88f6 100644
--- a/internal/console/authcli.go
+++ b/internal/console/authcli.go
@@ -716,6 +716,9 @@ func (s *Server) registerCLIRoutes() {
if s.cfg.MovesEnabled {
s.registerMovesPhase1Routes()
}
+ // The offline operator move's doors: always registered, operator-only,
+ // and refused with MV-O20 on a console with host claims or Phase 1 on.
+ s.registerOfflineMoveRoutes()
s.cliRoute("DELETE /api/cli/apps/{account}/{app}/instances/{host}", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceDelete)
// The import door (seeded-instances spec §5): the instance role, like
// the create door that begins it.