amadan

carlos / platform Public

offline-move: Task 3 fix round 1 (keep heals, strict listing, terminal left, capability from the worker)

- M1: reconcileSerializedWith reads the keep view again when its last
  listing failed, not only when unbuilt, so a failed startup listing
  heals on the next reconcile.
- M2: the dormant pin shows the very first reconcile on an unbuilt view
  removing a retired row exactly as main does, with a legacy and a
  Phase 1 move present.
- M3: a park rename that fails with EXDEV, EBUSY, EROFS, EPERM or EACCES
  is reported left ("cannot be renamed: ...") once, read back from the
  box's own report, and never retried: no re-fence or lift churn, and
  the report stays parked. Any other rename error reports failed and
  retries.
- M4: the view is built from ListAllRecordsStrict (startup and every
  moves pass); a host whose record does not decode is kept by keep
  only, and logged.
- M5: offline-move-1 is published per pass from the armed worker's
  OfflineCapable (with its view), the condition the reconcile hooks use.
- hostSliceCgroup is pinned to instancectl's slice.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 570194eb6e2d6cad8d8b0088db0eeb82e5457eba parent df9df63
5 files changed, +460 −119
  • internal/host/agent.go +5 −4
  • internal/host/edgesfile.go +20 −9
  • internal/host/movesoffline.go +132 −25
  • internal/host/movesoffline_test.go +299 −79
  • internal/host/movespass.go +4 −2
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 0faf0cd02..06c3f9c5e 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -1481,11 +1481,12 @@ func reconcileSerializedWith(ctx context.Context, s release.Store, reg *registry
// Unchanged with the Phase 1 switch off.
cfg = withMovesHolds(cfg)
// Offline move holds: the hold from the record alone, the keep from the
- // view, which is read here first if nothing has read it yet, so keep
- // never fails closed on a box whose listing reads. Nothing on a box that
- // cannot take part, and nothing where Phase 1's hooks are set.
+ // view, which is read here first if no listing has been read yet or the
+ // last one failed, so keep never fails closed on a box whose listing
+ // reads. Nothing on a box that cannot take part, and nothing where
+ // Phase 1's hooks are set.
if w != nil && w.cfg.OfflineCapable && w.cfg.Offline != nil {
- if !w.cfg.Offline.built() {
+ if !w.cfg.Offline.built() || w.cfg.Offline.failed() {
refreshOfflineView(ctx, s, w.cfg.Offline)
}
cfg = withOfflineHolds(cfg, offlineHold(w.cfg), w.cfg.Offline)
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go
index d39f3f6be..150f394a0 100644
--- a/internal/host/edgesfile.go
+++ b/internal/host/edgesfile.go
@@ -163,18 +163,25 @@ type edgeHeadroom struct {
const capRegionHA1 = "region-ha-1"
// edgesFileCapabilities publishes region-ha-1 only when CARLOS_REGION_HA is
-// set to something other than "" or "0", and offline-move-1 only when the
-// box can take part in an offline operator move (offlineCapable:
-// hibernation configured, CARLOS_HOST_CLAIM and CARLOS_MOVES_PHASE1 unset).
+// set to something other than "" or "0". offline-move-1 is added per pass
+// by withOfflineCapability.
func edgesFileCapabilities(getenv func(string) string) []string {
- var caps []string
- if regionHAEnabled(getenv) {
- caps = append(caps, capRegionHA1)
+ if !regionHAEnabled(getenv) {
+ return nil
}
- if offlineCapable(getenv, hibernationConfigured(getenv)) {
- caps = append(caps, capOfflineMove1)
+ return []string{capRegionHA1}
+}
+
+// withOfflineCapability is caps with offline-move-1 when capable. The loop
+// passes armedOfflineCapable(): the armed moves worker's own answer, the one
+// the reconcile hooks act on, so a box never publishes the capability
+// without the hold that makes it safe (a target that published it with no
+// armed worker would mint a landing row ready). caps is not modified.
+func withOfflineCapability(caps []string, capable bool) []string {
+ if !capable {
+ return caps
}
- return caps
+ return append(slices.Clone(caps), capOfflineMove1)
}
// headroomCapabilities is what headroomFor is asked about: region HA's
@@ -832,6 +839,9 @@ func edgesFileLoop(reg *registry.Registry) {
BoxID: box,
Capabilities: edgesFileCapabilities(os.Getenv),
}
+ // The flag-derived capabilities; offline-move-1 joins them on each pass
+ // once the moves worker is armed and capable.
+ baseCaps := ident.Capabilities
dataDir := provisionTickConfig(os.Getenv).DataDir
optDir := release.OptDir()
// The local floor files this file publishes are refreshed hourly by
@@ -876,6 +886,7 @@ func edgesFileLoop(reg *registry.Registry) {
if state.timed() {
eip = readPublicIP()
}
+ ident.Capabilities = withOfflineCapability(baseCaps, armedOfflineCapable())
ident.Headroom = headroomFor(headroomCapabilities(ident.Capabilities), dataDir, reg)
ident = withMovesFields(ident, dataDir, optDir)
ident = withSeedsFields(ident, dataDir, optDir)
diff --git a/internal/host/movesoffline.go b/internal/host/movesoffline.go
index 1fef1c720..9b1832ef6 100644
--- a/internal/host/movesoffline.go
+++ b/internal/host/movesoffline.go
@@ -11,7 +11,9 @@ import (
"path/filepath"
"sort"
"strings"
+ "sync"
"sync/atomic"
+ "syscall"
"time"
"github.com/carlosframework/platform/internal/activator"
@@ -119,10 +121,11 @@ func offlineHold(box moveCfg) func(rec provision.Record) (status, op string, ok
//
// It is refreshed from every moves pass's record list and once at agent
// start, before the reconcile loops (refreshOfflineView). keep fails closed,
-// keeping every row, only while no listing has been read: when the startup
-// listing failed, and in the moment before it (reconcileSerializedWith reads
-// one first when the view is unbuilt). Once a listing is read, a host with
-// no offline move is not kept, so removals happen on the first pass as they
+// keeping every row, only while no listing has been read (reconcileSerialized-
+// With reads one first when the view is unbuilt or its last listing failed,
+// so a failed startup listing heals on the next reconcile), and for a host
+// whose record does not decode. Once a listing is read, a host with no
+// offline move is not kept, so removals happen on the first pass as they
// always have.
type offlineView struct {
home, region string
@@ -131,11 +134,16 @@ type offlineView struct {
// contract, G10). Nil writes nothing.
written func(host string)
snap atomic.Pointer[offlineSnapshot]
+ logged sync.Map // host -> time.Time: the last unreadable-record line
}
type offlineSnapshot struct {
moves map[string]provision.MoveState // host -> its offline move
failed bool // the listing failed: keep everything
+ // unreadable are the hosts whose record read but did not decode, or
+ // disagreed with its path (provision.ListAllRecordsStrict): their move
+ // cannot be known, so their rows are kept.
+ unreadable map[string]bool
}
// newOfflineView builds the view for the box cfg describes. Unbuilt until
@@ -144,14 +152,25 @@ func newOfflineView(cfg moveCfg, written func(host string)) *offlineView {
return &offlineView{home: cfg.ParkHomeRegion, region: cfg.parkRegion(), written: written}
}
-// refresh rebuilds the view from a record list that was read.
-func (v *offlineView) refresh(recs []provision.Record) {
- next := &offlineSnapshot{moves: map[string]provision.MoveState{}}
+// refresh rebuilds the view from a record list that was read, with the
+// hosts provision.ListAllRecordsStrict could not decode: those are kept
+// (fail closed for keep only; the hold never reads the view) and logged at
+// most once a minute each.
+func (v *offlineView) refresh(recs []provision.Record, unreadable map[string]bool) {
+ next := &offlineSnapshot{moves: map[string]provision.MoveState{}, unreadable: map[string]bool{}}
for _, rec := range recs {
if rec.Move.Offline() {
next.moves[rec.Host] = *rec.Move
}
}
+ now := time.Now()
+ for host := range unreadable {
+ next.unreadable[host] = true
+ if last, ok := v.logged.Load(host); !ok || now.Sub(last.(time.Time)) >= time.Minute {
+ v.logged.Store(host, now)
+ log.Printf("moves: offline view: %s: the record does not decode; its row is kept until it does", host)
+ }
+ }
v.snap.Store(next)
}
@@ -162,6 +181,13 @@ func (v *offlineView) fail() { v.snap.Store(&offlineSnapshot{failed: true}) }
// built reports whether any listing, read or failed, has been recorded.
func (v *offlineView) built() bool { return v.snap.Load() != nil }
+// failed reports whether the last listing recorded failed: keep answers true
+// for every host until one reads.
+func (v *offlineView) failed() bool {
+ snap := v.snap.Load()
+ return snap != nil && snap.failed
+}
+
// keep is reconcile's KeepParticipant: the row of a host whose offline move
// names this box's region stays, whatever the record's region says. The
// source keeps it until offline-finishing (when it is removed, as the plan's
@@ -169,7 +195,7 @@ func (v *offlineView) built() bool { return v.snap.Load() != nil }
// (the target's revert deletes it itself). An unknown state keeps.
func (v *offlineView) keep(host string) bool {
snap := v.snap.Load()
- if snap == nil || snap.failed {
+ if snap == nil || snap.failed || snap.unreadable[host] {
return true
}
m, ok := snap.moves[host]
@@ -192,13 +218,13 @@ func refreshOfflineView(ctx context.Context, s release.Store, v *offlineView) {
if v == nil || s == nil {
return
}
- recs, err := provision.ListAllRecords(ctx, s)
+ recs, unreadable, err := provision.ListAllRecordsStrict(ctx, s)
if err != nil {
log.Printf("moves: offline view: list records: %v; every row is kept until a listing reads", err)
v.fail()
return
}
- v.refresh(recs)
+ v.refresh(recs, unreadable)
}
// armedOfflineView is the armed worker's view, nil when no worker is armed
@@ -211,6 +237,11 @@ func armedOfflineView() *offlineView {
return w.cfg.Offline
}
+// armedOfflineCapable is whether this box publishes offline-move-1: the
+// armed worker is capable and has its keep view, exactly the condition
+// under which reconcileSerializedWith installs the offline hooks.
+func armedOfflineCapable() bool { return armedOfflineView() != nil }
+
// withOfflineHolds gives a reconcile pass the offline hooks, only when the
// caller and Phase 1's view set none (withMovesHolds runs first): the two
// are never armed together, since a box with Phase 1 on is not capable.
@@ -724,39 +755,96 @@ func parkName(dir, name string) string {
}
}
+// Why a rename the park attempted did not happen. A path whose rename fails
+// with an error a later pass would meet again (a mount, EXDEV, EBUSY; a path
+// the agent may not rename, EPERM or EACCES; a read-only filesystem) is
+// left for good: reported once, with leftCannot and the error, and never
+// tried again for this move, so a pass does not re-fence, fail and lift
+// over it. Any other error is leftNotYet, and the next pass tries again.
+const (
+ leftCannot = "cannot be renamed: "
+ leftNotYet = "not renamed yet: "
+)
+
+// renameTerminal reports whether a rename error is one a later pass would
+// meet again.
+func renameTerminal(err error) bool {
+ return errors.Is(err, os.ErrPermission) || errors.Is(err, syscall.EXDEV) ||
+ errors.Is(err, syscall.EBUSY) || errors.Is(err, syscall.EROFS)
+}
+
// retire reports one path's action from what is on disk, renaming it into
// the park directory first when park is true and it is there to rename.
-// Nothing is ever deleted.
-func retire(p retirePath, parkDir string, park bool) moves.RetiredPath {
+// cannot is the reason an earlier pass recorded for a path it found it can
+// never rename ("" when none): such a path is reported left again, as it
+// is, and not tried. Nothing is ever deleted.
+func retire(p retirePath, parkDir string, park bool, cannot string) moves.RetiredPath {
present := lexists(p.path)
switch {
case p.left != "" && present:
return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: p.left}
case p.left != "":
return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+ case present && cannot != "":
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: cannot}
case present && park:
dst := parkName(parkDir, p.name)
- if err := os.MkdirAll(filepath.Dir(dst), 0o700); err != nil {
- return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: "not renamed: " + err.Error()}
- }
- if err := os.Rename(p.path, dst); err != nil {
- return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: "not renamed: " + err.Error()}
- }
- return moves.RetiredPath{Path: p.path, Action: moves.RetiredParked}
+ err := os.MkdirAll(filepath.Dir(dst), 0o700)
+ if err == nil {
+ err = os.Rename(p.path, dst)
+ }
+ switch {
+ case err == nil:
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredParked}
+ case errors.Is(err, os.ErrNotExist) && !lexists(p.path):
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
+ case renameTerminal(err):
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: leftCannot + err.Error()}
+ }
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: leftNotYet + err.Error()}
case present:
- return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: "not renamed: the stop could not be confirmed this pass"}
+ return moves.RetiredPath{Path: p.path, Action: moves.RetiredLeft, Reason: leftNotYet + "the stop could not be confirmed this pass"}
case lexists(filepath.Join(parkDir, p.name)):
return moves.RetiredPath{Path: p.path, Action: moves.RetiredParked}
}
return moves.RetiredPath{Path: p.path, Action: moves.RetiredAbsent}
}
+// cannotRename reads this box's own last report for the move and returns
+// the paths it recorded as never renameable (leftCannot), with their
+// reasons. Reading them from the report, not from memory, keeps the answer
+// across an agent restart.
+func (w *movesWorker) cannotRename(ctx context.Context, s release.Store, host, op string) (map[string]string, error) {
+ out := map[string]string{}
+ body, _, err := s.Get(ctx, moves.OfflineReportKey(host, op, moves.OfflineRoleSource, w.cfg.Box))
+ if errors.Is(err, os.ErrNotExist) {
+ return out, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ var r moves.OfflineReport
+ if err := json.Unmarshal(body, &r); err != nil {
+ return nil, err
+ }
+ for _, rp := range r.Retired {
+ if rp.Action == moves.RetiredLeft && strings.HasPrefix(rp.Reason, leftCannot) {
+ out[rp.Path] = rp.Reason
+ }
+ }
+ return out, nil
+}
+
// offlineSourcePark is the source step at offline-open: with the fence held,
// the row held (or absent), the host quiesced, no process left and no layout
// journal, rename every path of the source-retire row that is still here
// into the park directory; then lift the fence and report parked with the
// inventory. A pass that finds nothing left to rename does not fence again:
-// it reports from what is parked. Nothing is deleted.
+// it reports from what is parked. A path that can never be renamed
+// (leftCannot) is reported left with its reason and does not count as left
+// to rename, so it neither blocks the parked report nor brings the fence
+// back on every pass. A rename that may yet succeed (leftNotYet) reports
+// failed, and the next pass tries again. Nothing is deleted.
func (w *movesWorker) offlineSourcePark(ctx context.Context, s release.Store, rec provision.Record) error {
cfg := w.cfg
m, host := rec.Move, rec.Host
@@ -772,9 +860,13 @@ func (w *movesWorker) offlineSourcePark(ctx context.Context, s release.Store, re
if err != nil {
return err
}
+ cannot, err := w.cannotRename(ctx, s, host, m.ID)
+ if err != nil {
+ return err
+ }
pending := false
for _, p := range paths {
- if p.left == "" && lexists(p.path) {
+ if p.left == "" && cannot[p.path] == "" && lexists(p.path) {
pending = true
break
}
@@ -793,16 +885,31 @@ func (w *movesWorker) offlineSourcePark(ctx context.Context, s release.Store, re
}
}
retired := make([]moves.RetiredPath, 0, len(paths))
+ retry := false
for _, p := range paths {
- retired = append(retired, retire(p, parkDir, pending))
+ rp := retire(p, parkDir, pending, cannot[p.path])
+ if rp.Action == moves.RetiredLeft && strings.HasPrefix(rp.Reason, leftNotYet) {
+ retry = true
+ }
+ retired = append(retired, rp)
}
w.offlineLiftFence(host, m)
- r, err := w.sourceReport(host, m, rt, false, moves.OfflinePhaseParked)
+ phase := moves.OfflinePhaseParked
+ if retry {
+ phase = moves.OfflinePhaseFailed
+ }
+ r, err := w.sourceReport(host, m, rt, false, phase)
if err != nil {
return err
}
r.Retired = retired
- return writeOfflineReport(ctx, s, r)
+ if err := writeOfflineReport(ctx, s, r); err != nil {
+ return err
+ }
+ if retry {
+ return fmt.Errorf("offline move %s: %s: some paths were not renamed this pass; the next pass tries again", m.ID, host)
+ }
+ return nil
}
// offlineSourceGone is the source step at offline-finishing: keep no longer
diff --git a/internal/host/movesoffline_test.go b/internal/host/movesoffline_test.go
index 191a73fe8..b1b8da306 100644
--- a/internal/host/movesoffline_test.go
+++ b/internal/host/movesoffline_test.go
@@ -19,6 +19,7 @@ import (
"github.com/carlosframework/platform/internal/cgroup"
"github.com/carlosframework/platform/internal/domains"
"github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/instancectl"
"github.com/carlosframework/platform/internal/layout"
"github.com/carlosframework/platform/internal/moves"
"github.com/carlosframework/platform/internal/provision"
@@ -339,7 +340,7 @@ func TestOfflineFreshTargetMintsTheLandingRowHeld(t *testing.T) {
wd.flip()
uk := wd.box("uk-box", "uk", nil)
- uk.w.cfg.Offline.refresh(nil) // stale: no offline move in it
+ uk.w.cfg.Offline.refresh(nil, nil) // stale: no offline move in it
uk.reconcile()
rt := uk.row(offHost)
if rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
@@ -706,6 +707,88 @@ func assertRetired(t *testing.T, got []moves.RetiredPath, want map[string]string
}
}
+// M3: a path the agent can never rename (here a file in a directory it may
+// not write, as a root-owned /data/programs is) is reported left once, with
+// its reason, and is not tried again: later passes, an agent restart
+// included, neither fence nor rename, and the report stays parked.
+func TestOfflineParkLeavesWhatCannotBeRenamedOnce(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("root can rename anything")
+ }
+ wd := newOffWorld(t)
+ a := wd.box("ie-a", "", nil)
+ a.tick()
+ db := a.row(offHost).DBPath
+ writeFile(t, db, "sqlite")
+ programs := filepath.Join(a.dd, "programs")
+ writeFile(t, filepath.Join(programs, offHost+".json"), "{}")
+ if err := os.Chmod(programs, 0o555); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Chmod(programs, 0o755) })
+ fences := 0
+ count := func() {
+ inner := a.w.cfg.FenceAs
+ a.w.cfg.FenceAs = func(ctx context.Context, host string, gen uint64, owner string) error {
+ fences++
+ return inner(ctx, host, gen, owner)
+ }
+ }
+ wd.start()
+ a.tick()
+ wd.seal()
+ wd.sealed()
+ wd.flip()
+ wd.landed()
+ a.tick()
+ wd.open()
+ count()
+ before := fences
+ a.tick()
+ if fences != before+1 {
+ t.Fatalf("the first park fenced %d times", fences-before)
+ }
+ r := wd.report(moves.OfflineRoleSource, "ie-a")
+ if r.Phase != moves.OfflinePhaseParked {
+ t.Fatalf("phase %s", r.Phase)
+ }
+ var left moves.RetiredPath
+ for _, rp := range r.Retired {
+ if rp.Path == filepath.Join(programs, offHost+".json") {
+ left = rp
+ }
+ if rp.Path == db && rp.Action != moves.RetiredParked {
+ t.Fatalf("the DB: %+v", rp)
+ }
+ }
+ if left.Action != moves.RetiredLeft || !strings.HasPrefix(left.Reason, leftCannot) {
+ t.Fatalf("the unrenameable path: %+v", left)
+ }
+ for i := 0; i < 2; i++ {
+ before = fences
+ a.tick()
+ if fences != before {
+ t.Fatal("a later pass fenced again for a path it cannot rename")
+ }
+ if again := wd.report(moves.OfflineRoleSource, "ie-a"); again.Phase != moves.OfflinePhaseParked || !reflect.DeepEqual(again.Retired, r.Retired) {
+ t.Fatalf("pass %d: %+v", i, again)
+ }
+ }
+ a.restart()
+ count()
+ before = fences
+ a.tick()
+ if fences != before || a.svc.Fenced(offHost) {
+ t.Fatal("after a restart the pass fenced again for a path it cannot rename")
+ }
+ if again := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(again.Retired, r.Retired) {
+ t.Fatalf("after a restart: %+v", again.Retired)
+ }
+ if _, err := os.Stat(filepath.Join(programs, offHost+".json")); err != nil {
+ t.Fatalf("the left path: %v", err)
+ }
+}
+
// offline-open on a nested source: the contents of the raw instance
// directory are renamed into the park directory, the directory and its
// view stay, empty, and the inventory names each path.
@@ -834,11 +917,28 @@ func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) {
}
return func(k string) string { return m[k] }
}
- if got := edgesFileCapabilities(env()); !slices.Equal(got, []string{capOfflineMove1}) {
+ // The published capability is the armed worker's own answer (M5).
+ prev := armedMovesWorker.Load()
+ t.Cleanup(func() { armedMovesWorker.Store(prev) })
+ armedMovesWorker.Store(nil)
+ if armedOfflineCapable() {
+ t.Fatal("capable with no worker armed")
+ }
+ if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); len(got) != 0 {
+ t.Fatalf("no worker armed: %v", got)
+ }
+ wdCap := newOffWorld(t)
+ armed := wdCap.box("armed", "", nil)
+ armedMovesWorker.Store(armed.w)
+ if !armedOfflineCapable() {
+ t.Fatal("a capable armed worker does not publish")
+ }
+ if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); !slices.Equal(got, []string{capOfflineMove1}) {
t.Fatalf("capable box: %v", got)
}
- if got := edgesFileCapabilities(env("CARLOS_REGION_HA", "1")); !slices.Equal(got, []string{capRegionHA1, capOfflineMove1}) {
- t.Fatalf("region HA too: %v", got)
+ base := edgesFileCapabilities(env("CARLOS_REGION_HA", "1"))
+ if got := withOfflineCapability(base, true); !slices.Equal(got, []string{capRegionHA1, capOfflineMove1}) || !slices.Equal(base, []string{capRegionHA1}) {
+ t.Fatalf("region HA too: %v (base %v)", got, base)
}
for _, kv := range [][]string{
{"CARLOS_HIBERNATE_BUCKET", ""},
@@ -847,13 +947,16 @@ func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) {
{moves.SwitchEnv, "on"},
{moves.SwitchEnv, "off"}, // set at all
} {
- if got := edgesFileCapabilities(env(kv...)); slices.Contains(got, capOfflineMove1) {
- t.Fatalf("%s=%q: %v", kv[0], kv[1], got)
- }
if offlineCapable(env(kv...), hibernationConfigured(env(kv...))) {
t.Fatalf("%s=%q is capable", kv[0], kv[1])
}
+ incapable := wdCap.box("incapable", "", map[string]string{kv[0]: kv[1]})
+ armedMovesWorker.Store(incapable.w)
+ if got := withOfflineCapability(edgesFileCapabilities(env(kv...)), armedOfflineCapable()); slices.Contains(got, capOfflineMove1) {
+ t.Fatalf("%s=%q publishes: %v", kv[0], kv[1], got)
+ }
}
+ armedMovesWorker.Store(prev)
wd := newOffWorld(t)
a := wd.box("ie-a", "", map[string]string{"CARLOS_HOST_CLAIM": "1"})
@@ -877,119 +980,146 @@ func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) {
}
}
-// --- dormant ----------------------------------------------------------------
+// --- dormant ---------------------------------------------------------------
-// With no offline record anywhere: the hold answers false for every record,
-// keep answers false once the startup listing is read (and true only while
-// none has been, or when it failed), a retired row is removed on the first
-// pass exactly as with no hooks at all, and the edges file differs by the
-// one capability string and nothing else.
-func TestOfflineDormantIsMain(t *testing.T) {
- ctx := context.Background()
+// flatLayout is a reconcile config's layout read, always flat.
+func flatLayout(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }
+
+// dormantWorld is a deployment with no offline record anywhere: offHost at
+// rest, gone.example.com (whose record the test deletes), a legacy move and
+// a Phase 1 move on records of their own.
+func dormantWorld(t *testing.T) *offWorld {
+ t.Helper()
wd := newOffWorld(t)
- wd.create("gone.example.com")
- // A Phase 1 and a legacy move on records of their own: no offline hook
- // answers for them either.
- wd.create("legacy.example.com")
+ for _, h := range []string{"gone.example.com", "legacy.example.com", "phase1.example.com"} {
+ wd.create(h)
+ }
wd.moveHost("legacy.example.com", func(rec *provision.Record) error {
rec.Move = &provision.MoveState{To: "uk", State: provision.MovePreparing}
return nil
})
+ wd.moveHost("phase1.example.com", func(rec *provision.Record) error {
+ rec.Move = &provision.MoveState{ID: offID2, From: "", To: "uk", State: provision.MovePreparing, RequestedAt: wd.clock()}
+ return nil
+ })
+ return wd
+}
- run := func(withHooks bool) *offBox {
- b := wd.box(map[bool]string{false: "main", true: "hooked"}[withHooks], "", nil)
- w := b.w
- if !withHooks {
- w = nil
- }
- cfg := provision.ReconcileConfig{Region: "", DataDir: b.dd, Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }}
- if err := reconcileSerializedWith(ctx, wd.s, b.reg, cfg, w); err != nil {
+// dormantBoxes are two home boxes minted from the same records with no
+// hooks at all, so their registries start equal: the plain one stands for
+// main, the hooked one has a capable worker whose view no listing has read.
+func dormantBoxes(t *testing.T, wd *offWorld) (plain, hooked *offBox) {
+ t.Helper()
+ plain, hooked = wd.box("main", "", nil), wd.box("hooked", "", nil)
+ for _, b := range []*offBox{plain, hooked} {
+ if err := reconcileSerializedWith(context.Background(), wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, nil); err != nil {
t.Fatal(err)
}
- return b
+ if b.row("gone.example.com") == nil {
+ t.Fatalf("fixture: %s has no row to retire", b.name)
+ }
+ }
+ if hooked.w.cfg.Offline.built() || !hooked.w.cfg.OfflineCapable {
+ t.Fatal("fixture: the hooked box's view is built, or the box is not capable")
}
- plainBox := run(false)
- hooked := run(true)
+ return plain, hooked
+}
- // The startup refresh: keep is false for every host.
- refreshOfflineView(ctx, wd.s, hooked.w.cfg.Offline)
- for _, h := range []string{offHost, "gone.example.com", "legacy.example.com", "never.example.com"} {
- if hooked.w.cfg.Offline.keep(h) {
- t.Fatalf("keep(%s) after the startup refresh", h)
+// sameRegistries fails unless the two boxes' registries match, paths and
+// creation times aside (each box has its own data directory and clock).
+func sameRegistries(t *testing.T, a, b *offBox) {
+ t.Helper()
+ strip := func(rs []registry.Route) []registry.Route {
+ out := append([]registry.Route(nil), rs...)
+ for i := range out {
+ out[i].DBPath, out[i].Socket, out[i].CreatedAt = "", "", 0
}
+ return out
}
- recs, err := provision.ListAllRecords(ctx, wd.s)
+ ar, err := a.reg.List()
if err != nil {
t.Fatal(err)
}
- for _, rec := range recs {
- if _, _, ok := offlineHold(hooked.w.cfg)(rec); ok {
- t.Fatalf("held %s", rec.Host)
- }
+ br, err := b.reg.List()
+ if err != nil {
+ t.Fatal(err)
}
+ if !reflect.DeepEqual(strip(ar), strip(br)) {
+ t.Fatalf("the registries differ:\n %s %+v\n %s %+v", a.name, ar, b.name, br)
+ }
+}
- // The record of gone.example.com goes; the first pass removes its row
- // on both boxes alike.
+// With no offline record anywhere (a legacy and a Phase 1 move included):
+// the very first reconcile on a box whose view no listing has read removes
+// a retired row exactly as main does; after it the hold answers false for
+// every record and keep false for every host; and the edges file differs
+// by the one capability string and nothing else.
+func TestOfflineDormantIsMain(t *testing.T) {
+ ctx := context.Background()
+ wd := dormantWorld(t)
+ plain, hooked := dormantBoxes(t, wd)
if err := wd.s.Delete(ctx, provision.RecordKey(wd.acct, "app", "gone.example.com")); err != nil {
t.Fatal(err)
}
- for _, b := range []*offBox{plainBox, hooked} {
+ for _, b := range []*offBox{plain, hooked} {
w := b.w
- if b == plainBox {
+ if b == plain {
w = nil
}
- cfg := provision.ReconcileConfig{Region: "", DataDir: b.dd, Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }}
- if err := reconcileSerializedWith(ctx, wd.s, b.reg, cfg, w); err != nil {
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
t.Fatal(err)
}
if rt := b.row("gone.example.com"); rt != nil {
- t.Fatalf("%s kept a retired row: %+v", b.name, rt)
+ t.Fatalf("%s kept a retired row on its first pass: %+v", b.name, rt)
}
}
- strip := func(rs []registry.Route) []registry.Route {
- out := append([]registry.Route(nil), rs...)
- for i := range out {
- out[i].DBPath, out[i].Socket, out[i].CreatedAt = "", "", 0
+ sameRegistries(t, plain, hooked)
+ v := hooked.w.cfg.Offline
+ if !v.built() || v.failed() {
+ t.Fatal("the first reconcile did not read the view")
+ }
+ for _, h := range []string{offHost, "gone.example.com", "legacy.example.com", "phase1.example.com", "never.example.com"} {
+ if v.keep(h) {
+ t.Fatalf("keep(%s)", h)
}
- return out
}
- mr, _ := plainBox.reg.List()
- hr, _ := hooked.reg.List()
- if !reflect.DeepEqual(strip(mr), strip(hr)) {
- t.Fatalf("the registries differ:\n main %+v\n hooked %+v", mr, hr)
+ recs, err := provision.ListAllRecords(ctx, wd.s)
+ if err != nil {
+ t.Fatal(err)
}
-
- // keep fails closed only with no listing read: never built, or failed.
- unbuilt := newOfflineView(moveCfg{Region: ""}, nil)
- if !unbuilt.keep(offHost) {
- t.Fatal("an unbuilt view keeps nothing")
+ if len(recs) != 3 {
+ t.Fatalf("fixture: %d records", len(recs))
}
- refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, unbuilt)
- if !unbuilt.built() || !unbuilt.keep(offHost) {
- t.Fatal("a failed startup listing does not keep")
+ for _, rec := range recs {
+ if _, _, ok := offlineHold(hooked.w.cfg)(rec); ok {
+ t.Fatalf("held %s", rec.Host)
+ }
}
- unbuilt.refresh(recs)
- if unbuilt.keep(offHost) {
- t.Fatal("a listing read after a failure still keeps")
+ // A further pass changes nothing on either box.
+ for _, b := range []*offBox{plain, hooked} {
+ w := b.w
+ if b == plain {
+ w = nil
+ }
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
+ t.Fatal(err)
+ }
}
+ sameRegistries(t, plain, hooked)
// The edges file: the one string, and no headroom with it.
var last1, last2 []byte
- plain := edgesFile{Box: "plain", Placement: ""}
- capable := edgesFile{Box: "plain", Placement: "", Capabilities: edgesFileCapabilities(func(k string) string {
- if k == "CARLOS_HIBERNATE_BUCKET" {
- return "data"
- }
- return ""
- })}
+ none := func(string) string { return "" }
+ plainFile := edgesFile{Box: "plain", Placement: "", Capabilities: withOfflineCapability(edgesFileCapabilities(none), false)}
+ capable := edgesFile{Box: "plain", Placement: "", Capabilities: withOfflineCapability(edgesFileCapabilities(none), true)}
if headroomCapabilities(capable.Capabilities) != nil {
t.Fatal("offline-move-1 asks for headroom")
}
s1, s2 := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
- if err := edgesFileWrite(ctx, s1, plainBox.reg, plain, "1.2.3.4", &last1, false); err != nil {
+ if err := edgesFileWrite(ctx, s1, plain.reg, plainFile, "1.2.3.4", &last1, false); err != nil {
t.Fatal(err)
}
- if err := edgesFileWrite(ctx, s2, plainBox.reg, capable, "1.2.3.4", &last2, false); err != nil {
+ if err := edgesFileWrite(ctx, s2, plain.reg, capable, "1.2.3.4", &last2, false); err != nil {
t.Fatal(err)
}
b1, _, _ := s1.Get(ctx, edgesFileKey("plain"))
@@ -1012,6 +1142,96 @@ func TestOfflineDormantIsMain(t *testing.T) {
}
}
+// M1: a failed startup listing keeps every row only until the next
+// reconcile, which reads the listing again: the retired row then goes as on
+// main, without waiting for a moves pass.
+func TestOfflineFailedListingHealsOnTheNextReconcile(t *testing.T) {
+ ctx := context.Background()
+ wd := dormantWorld(t)
+ plain, hooked := dormantBoxes(t, wd)
+ v := hooked.w.cfg.Offline
+ refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, v)
+ if !v.built() || !v.failed() || !v.keep(offHost) {
+ t.Fatal("a failed startup listing does not keep")
+ }
+ if err := wd.s.Delete(ctx, provision.RecordKey(wd.acct, "app", "gone.example.com")); err != nil {
+ t.Fatal(err)
+ }
+ for _, b := range []*offBox{plain, hooked} {
+ w := b.w
+ if b == plain {
+ w = nil
+ }
+ if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if v.failed() || v.keep("gone.example.com") {
+ t.Fatal("the reconcile did not read the listing again")
+ }
+ if rt := hooked.row("gone.example.com"); rt != nil {
+ t.Fatalf("the retired row stayed after the view healed: %+v", rt)
+ }
+ sameRegistries(t, plain, hooked)
+ // While the listing keeps failing the view stays closed.
+ refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, v)
+ if !v.keep(offHost) {
+ t.Fatal("a failed listing does not keep")
+ }
+ unbuilt := newOfflineView(moveCfg{Region: ""}, nil)
+ if !unbuilt.keep(offHost) {
+ t.Fatal("an unbuilt view keeps nothing")
+ }
+}
+
+// M4: a record that does not decode is kept by keep alone (its move cannot
+// be known), logged, and every other host is judged as usual. The moves
+// pass's listing carries the same answer.
+func TestOfflineUndecodableRecordIsKept(t *testing.T) {
+ ctx := context.Background()
+ wd := newOffWorld(t)
+ wd.create("broken.example.com")
+ if _, err := wd.s.Put(ctx, provision.RecordKey(wd.acct, "app", "broken.example.com"), []byte("{not json"), mustETag(t, wd.s, provision.RecordKey(wd.acct, "app", "broken.example.com"))); err != nil {
+ t.Fatal(err)
+ }
+ a := wd.box("ie-a", "", nil)
+ refreshOfflineView(ctx, wd.s, a.w.cfg.Offline)
+ v := a.w.cfg.Offline
+ if v.failed() || !v.keep("broken.example.com") || v.keep(offHost) {
+ t.Fatalf("startup view: failed %v, keep(broken) %v, keep(%s) %v", v.failed(), v.keep("broken.example.com"), offHost, v.keep(offHost))
+ }
+ fresh := newOfflineView(a.w.cfg, nil)
+ a.w.cfg.Offline = fresh
+ a.pass()
+ if !fresh.keep("broken.example.com") || fresh.keep(offHost) {
+ t.Fatal("the moves pass's listing does not keep the undecodable host")
+ }
+ if _, _, ok := offlineHold(a.w.cfg)(provision.Record{Host: "broken.example.com"}); ok {
+ t.Fatal("held a record with no move")
+ }
+}
+
+func mustETag(t *testing.T, s release.Store, key string) string {
+ t.Helper()
+ _, etag, err := s.Get(context.Background(), key)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return etag
+}
+
+// hostSliceCgroup is the slice instancectl's units run in.
+func TestOfflineHostSliceIsInstancectlsSlice(t *testing.T) {
+ got := hostSliceCgroup("/sys/fs/cgroup", offHost)
+ want := filepath.Join("/sys/fs/cgroup", "carlos.slice", "carlos-instances.slice", instancectl.SliceName(offHost))
+ if got != want {
+ t.Fatalf("hostSliceCgroup = %s, want %s", got, want)
+ }
+ if filepath.Dir(filepath.Join("/sys/fs/cgroup", filepath.FromSlash(instancectl.ClaimUnitControlGroup(offHost)))) != got {
+ t.Fatal("the run unit is not in the slice")
+ }
+}
+
// The seal record's writer is create-only: the first body stays, a re-run
// gets it back and writes nothing, and the report writer beside it replaces.
func TestOfflineCreateOnlyKeepsTheFirstWrite(t *testing.T) {
@@ -1045,17 +1265,17 @@ func TestOfflineKeep(t *testing.T) {
for _, st := range states {
recs := []provision.Record{rec("uk", st)}
src := newOfflineView(moveCfg{Region: ""}, nil)
- src.refresh(recs)
+ src.refresh(recs, nil)
if got, want := src.keep(offHost), st != provision.OfflineFinishing; got != want {
t.Fatalf("source at %s: keep %v", st, got)
}
tgt := newOfflineView(moveCfg{Region: "uk"}, nil)
- tgt.refresh(recs)
+ tgt.refresh(recs, nil)
if !tgt.keep(offHost) {
t.Fatalf("target at %s: not kept", st)
}
other := newOfflineView(moveCfg{Region: "fr"}, nil)
- other.refresh(recs)
+ other.refresh(recs, nil)
if other.keep(offHost) {
t.Fatalf("a third region keeps at %s", st)
}
diff --git a/internal/host/movespass.go b/internal/host/movespass.go
index b28e99d94..16efd306e 100644
--- a/internal/host/movespass.go
+++ b/internal/host/movespass.go
@@ -1074,7 +1074,9 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
w.flushPageOutbox(ctx)
w.flushLogOutbox(ctx, s)
}
- recs, err := provision.ListAllRecords(ctx, s)
+ // Strict returns exactly ListAllRecords' records and errors, plus the
+ // hosts whose record did not decode, which the offline keep view keeps.
+ recs, unreadable, err := provision.ListAllRecordsStrict(ctx, s)
if err != nil {
return fmt.Errorf("moves: list records: %w", err)
}
@@ -1100,7 +1102,7 @@ func movesPass(ctx context.Context, s release.Store, w *movesWorker) error {
}
// The offline keep view, from the list this pass read (movesoffline.go).
if cfg.Offline != nil {
- cfg.Offline.refresh(recs)
+ cfg.Offline.refresh(recs, unreadable)
}
for _, rec := range recs {
// An offline operator move is routed first, ahead of the Phase 1