amadan
layout: SetInstancePaths, Fenced/StopUncertain, the fleet layout intent log (plan 4, task 1)
Adds the CAS write the layout pass uses to move a route between the flat
and nested storage layout, two read-only activator fence queries the pass
needs before treating an unproven stop as safe, and the append-log object
(control/layout.json) that carries the fleet's layout intent. Per the
2026-09-25 amendments (R2, R7, R16): does not add Unfence, since
LiftFence already covers that release path; uses the
LayoutIntents{Schema, Requests} envelope rather than a single object, so a
box reads the newest request that applies to its own label; and writes
the newer-schema test fixture through Delete then Put rather than a raw
put the store does not have.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
6 files changed,
+414
−0
internal/activator/maintenance.go+29 −0internal/activator/maintenance_test.go+66 −0internal/registry/registry.go+35 −0internal/registry/registry_test.go+32 −0internal/release/layoutintent.go+166 −0internal/release/layoutintent_test.go+86 −0
diff --git a/internal/activator/maintenance.go b/internal/activator/maintenance.go| index ca1b1206a..1528be129 100644 |
| --- a/internal/activator/maintenance.go |
| +++ b/internal/activator/maintenance.go |
| @@ -169,6 +169,35 @@ func (a *Service) wakeAllowed(host string) bool { |
| defer a.mu.Unlock() |
| return a.fences[host] == 0 |
| } |
| + |
| +// Fenced reports whether a maintenance fence is installed for host right |
| +// now. It is the layout pass's own affirmative read of the same map |
| +// wakeAllowed checks, so a pass deciding whether it is safe to install its |
| +// own fence can ask the question the way it is phrased in its own logic, |
| +// rather than negating an admission check meant for something else. |
| +func (a *Service) Fenced(host string) bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + return a.fences[host] != 0 |
| +} |
| + |
| +// StopUncertain reports whether host's process state cannot yet be trusted |
| +// as stopped: either an entry for it is still live right now, in any state |
| +// (waking, awake or stopping), or the last recorded stop observation never |
| +// finished every check FenceAndStop requires for proof. A pass that meets |
| +// ErrStopUnproven for an entry-less host still needs this before it can tell |
| +// "nothing was ever running" apart from "something was running and its |
| +// closure was never confirmed"; only the first of those is safe to treat as |
| +// a completed stop. |
| +func (a *Service) StopUncertain(host string) bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + if a.entries[host] != nil { |
| + return true |
| + } |
| + outcome, ok := a.stopHistory[host] |
| + return ok && !outcome.complete() |
| +} |
| func (a *Service) rememberStopLocked(host string, observation *stopObservation) { |
| if a.stopHistory == nil { |
| a.stopHistory = make(map[string]*stopObservation) |
diff --git a/internal/activator/maintenance_test.go b/internal/activator/maintenance_test.go| index 8a78ed15f..12ae718df 100644 |
| --- a/internal/activator/maintenance_test.go |
| +++ b/internal/activator/maintenance_test.go |
| @@ -629,3 +629,69 @@ func TestMaintenanceFailedAcquireReleasesReturnedLease(t *testing.T) { |
| t.Fatal("failed acquisition became stop proof", err) |
| } |
| } |
| + |
| +// TestFencedReportsAnInstalledFenceUntilItIsLifted: Fenced is the layout |
| +// pass's own affirmative read of what FenceAndStop and LiftFence leave |
| +// behind, so it must agree with them at every step rather than invert |
| +// wakeAllowed's private sense of the same map. |
| +func TestFencedReportsAnInstalledFenceUntilItIsLifted(t *testing.T) { |
| + a, e, _, _, _ := maintenanceAwake(t) |
| + if a.Fenced(e.host) { |
| + t.Fatal("fence reported before one was installed") |
| + } |
| + if _, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 1); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !a.Fenced(e.host) { |
| + t.Fatal("fence not reported as installed") |
| + } |
| + if !a.LiftFence(e.host, 1) { |
| + t.Fatal("LiftFence reported nothing to release") |
| + } |
| + if a.Fenced(e.host) { |
| + t.Fatal("fence still reported after it was released") |
| + } |
| +} |
| + |
| +// TestStopUncertainCoversLiveEntriesAndIncompleteHistory: a pass deciding |
| +// whether ErrStopUnproven can be treated as a completed stop needs both |
| +// halves of this: an entry that is still live right now (waking, awake or |
| +// stopping), and a past stop observation that never finished every check. |
| +// It must answer false for a host it has never seen at all, since that is |
| +// not evidence of anything. |
| +func TestStopUncertainCoversLiveEntriesAndIncompleteHistory(t *testing.T) { |
| + a, e, main, _, _ := maintenanceAwake(t) |
| + |
| + if !a.StopUncertain(e.host) { |
| + t.Fatal("a live entry must read as stop-uncertain") |
| + } |
| + if a.StopUncertain("never-seen.example") { |
| + t.Fatal("a host with no entry and no history must not read as uncertain") |
| + } |
| + |
| + // Drive the entry through an ordinary stop that fails, which removes the |
| + // entry but leaves an incomplete observation in stopHistory. |
| + main.err = errors.New("stop refused") |
| + a.mu.Lock() |
| + a.beginStopLocked(e) |
| + done := e.stopped |
| + a.mu.Unlock() |
| + <-done |
| + if a.Owns(e.host) { |
| + t.Fatal("entry should have been forgotten once the stop finished") |
| + } |
| + if !a.StopUncertain(e.host) { |
| + t.Fatal("an incomplete stop observation must read as uncertain") |
| + } |
| + |
| + // A different host's clean stop completes its observation and leaves no |
| + // live entry either. StopUncertain must read false for it, proving the |
| + // predicate is per-host rather than sticky once anything has failed. |
| + clean, ce, _, _, _ := maintenanceAwake(t) |
| + if proof, err := clean.FenceAndStop(maintenanceContext(t, time.Second), ce.host, 1); err != nil || !proof.Matches(ce.host, 1) { |
| + t.Fatalf("checked clean stop: %v", err) |
| + } |
| + if clean.StopUncertain(ce.host) { |
| + t.Fatal("a complete stop observation with no live entry must not read as uncertain") |
| + } |
| +} |
diff --git a/internal/registry/registry.go b/internal/registry/registry.go| index e4d558d66..ba96979df 100644 |
| --- a/internal/registry/registry.go |
| +++ b/internal/registry/registry.go |
| @@ -12,6 +12,7 @@ package registry |
| import ( |
| "crypto/sha256" |
| "database/sql" |
| + "errors" |
| "fmt" |
| "os" |
| "regexp" |
| @@ -1170,6 +1171,40 @@ func (r *Registry) SetDBPath(host, from, to string) error { |
| return nil |
| } |
| +// ErrPathsChanged is SetInstancePaths's answer when the row's two path |
| +// columns no longer hold the values the caller compared against. |
| +var ErrPathsChanged = errors.New("registry: instance paths changed underneath this pass") |
| + |
| +// SetInstancePaths moves a row between the flat and the nested layout |
| +// (isolated-exec design §8.2 step 8): one UPDATE guarded by BOTH columns' |
| +// current values, so a row that was migrated, reverted or re-added between |
| +// the pass's read and its write is left exactly as it is, and the caller |
| +// learns what it holds now. It is SetDBPath's sibling; the two columns are |
| +// swapped together because a row with one flat path and one nested path is |
| +// neither layout, and every consumer would refuse it. |
| +func (r *Registry) SetInstancePaths(host, fromDB, fromSock, toDB, toSock string) error { |
| + res, err := r.sql.Exec(`UPDATE routes SET db_path = ?, socket = ? WHERE host = ? AND db_path = ? AND socket = ?`, toDB, toSock, host, fromDB, fromSock) |
| + if err != nil { |
| + return err |
| + } |
| + n, err := res.RowsAffected() |
| + if err != nil { |
| + return err |
| + } |
| + if n == 0 { |
| + var curDB, curSock string |
| + serr := r.sql.QueryRow(`SELECT db_path, socket FROM routes WHERE host = ?`, host).Scan(&curDB, &curSock) |
| + if serr == sql.ErrNoRows { |
| + return fmt.Errorf("no route for host %q", host) |
| + } |
| + if serr != nil { |
| + return serr |
| + } |
| + return fmt.Errorf("%w: %s now holds db %q socket %q", ErrPathsChanged, host, curDB, curSock) |
| + } |
| + return nil |
| +} |
| + |
| // SetBacking stamps who owns a route's process: "" (an exec child of the |
| // agent) or BackingUnit (a carlos-app@<host> systemd unit). It is the door |
| // the app-capabilities migration (design §5b, §5c) needs and did not have: |
diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go| index 43353b324..b479ca6e1 100644 |
| --- a/internal/registry/registry_test.go |
| +++ b/internal/registry/registry_test.go |
| @@ -2,6 +2,7 @@ package registry |
| import ( |
| "database/sql" |
| + "errors" |
| "os" |
| "path/filepath" |
| "strings" |
| @@ -2799,3 +2800,34 @@ func mustRoute(t *testing.T, reg *Registry, host string) Route { |
| } |
| return *got |
| } |
| + |
| +// TestSetInstancePathsIsACompareAndSwapOnBothColumns: SetInstancePaths moves |
| +// a row between the flat and the nested layout by one UPDATE guarded by BOTH |
| +// path columns' current values, so a row a concurrent pass already moved, |
| +// reverted or re-added is left exactly as it is rather than overwritten. |
| +func TestSetInstancePathsIsACompareAndSwapOnBothColumns(t *testing.T) { |
| + r := openTestRegistry(t) |
| + if err := r.Upsert(Route{Host: "h.example.com", Kind: "instance", Socket: "/run/carlos/h.example.com.sock", DBPath: "/data/h.example.com.db"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := r.SetInstancePaths("h.example.com", "/data/h.example.com.db", "/run/carlos/h.example.com.sock", "/data/instances/h.example.com/h.example.com.db", "/run/carlos/instances/h.example.com/app.sock"); err != nil { |
| + t.Fatalf("CAS from the current values: %v", err) |
| + } |
| + rt := mustRoute(t, r, "h.example.com") |
| + if rt.DBPath != "/data/instances/h.example.com/h.example.com.db" || rt.Socket != "/run/carlos/instances/h.example.com/app.sock" { |
| + t.Fatalf("row after CAS = %q %q", rt.DBPath, rt.Socket) |
| + } |
| + // A second CAS from the OLD values misses on both columns and names the current ones. |
| + err := r.SetInstancePaths("h.example.com", "/data/h.example.com.db", "/run/carlos/h.example.com.sock", "/x", "/y") |
| + if !errors.Is(err, ErrPathsChanged) || !strings.Contains(err.Error(), "/data/instances/h.example.com/h.example.com.db") { |
| + t.Fatalf("stale CAS = %v", err) |
| + } |
| + // A CAS where only ONE column matches also misses: both must agree. |
| + err = r.SetInstancePaths("h.example.com", "/data/instances/h.example.com/h.example.com.db", "/run/carlos/h.example.com.sock", "/x", "/y") |
| + if !errors.Is(err, ErrPathsChanged) { |
| + t.Fatalf("half-matching CAS = %v", err) |
| + } |
| + if err := r.SetInstancePaths("absent.example.com", "a", "b", "c", "d"); err == nil || strings.Contains(err.Error(), "changed") { |
| + t.Fatalf("absent host = %v, want a no-route error", err) |
| + } |
| +} |
diff --git a/internal/release/layoutintent.go b/internal/release/layoutintent.go| new file mode 100644 |
| index 000000000..3f33ba3a7 |
| --- /dev/null |
| +++ b/internal/release/layoutintent.go |
| @@ -0,0 +1,166 @@ |
| +// The fleet's layout intent (isolated-exec design §8.2): the append log that |
| +// tells a box which storage layout, flat or nested, a route should converge |
| +// to. It records a MODE, not a one-shot request, so a row that becomes |
| +// eligible after the intent was recorded is migrated on the next pass, and a |
| +// box that was away converges when it returns. An old box never reads the |
| +// key; an old console never writes it. |
| +// |
| +// The shape mirrors restart-scope.json (see restartscope.go): a small |
| +// envelope carrying a schema and a log of requests, oldest first, so two |
| +// requests issued within one poll window cannot collapse into a single etag |
| +// move and lose one of them. |
| +package release |
| + |
| +import ( |
| + "context" |
| + "crypto/rand" |
| + "encoding/hex" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "io" |
| + "os" |
| + "time" |
| +) |
| + |
| +const ( |
| + LayoutModeNone = "" |
| + LayoutModeFlat = "flat" |
| + LayoutModeNested = "nested" |
| +) |
| + |
| +// LayoutScope narrows an intent to one box by label; an empty Box is the |
| +// whole fleet. |
| +type LayoutScope struct { |
| + Box string `json:"box,omitempty"` |
| +} |
| + |
| +// LayoutIntent is one recorded request in control/layout.json: which layout |
| +// to converge to, and which box or boxes it addresses. |
| +type LayoutIntent struct { |
| + ID string `json:"id"` |
| + RequestedAt string `json:"requested_at"` |
| + Mode string `json:"mode"` |
| + Scope LayoutScope `json:"scope"` |
| +} |
| + |
| +// LayoutIntents is the object at control/layout.json: the append log of |
| +// LayoutIntent requests, oldest first. |
| +type LayoutIntents struct { |
| + Schema int `json:"schema"` // 1 |
| + Requests []LayoutIntent `json:"requests"` |
| +} |
| + |
| +// LayoutIntentKey is the bucket key of the fleet's layout intent log. |
| +func LayoutIntentKey() string { return "control/layout.json" } |
| + |
| +// ValidLayoutMode reports whether m is a mode a box can converge to. The |
| +// empty mode is a valid ABSENCE (LayoutModeNone), never a valid request. |
| +func ValidLayoutMode(m string) bool { return m == LayoutModeFlat || m == LayoutModeNested } |
| + |
| +// Applies reports whether the intent addresses the box named boxLabel: every |
| +// box when Scope.Box is empty, or only the one box it names. |
| +func (in *LayoutIntent) Applies(boxLabel string) bool { |
| + return in != nil && (in.Scope.Box == "" || in.Scope.Box == boxLabel) |
| +} |
| + |
| +func parseLayoutIntents(key string, raw []byte) (*LayoutIntents, error) { |
| + var g LayoutIntents |
| + if err := json.Unmarshal(raw, &g); err != nil { |
| + return nil, fmt.Errorf("layout intent at %s: %w", key, err) |
| + } |
| + if g.Schema > 1 { |
| + return nil, fmt.Errorf("layout intent at %s: schema %d is newer than this binary understands; upgrade carlos", key, g.Schema) |
| + } |
| + return &g, nil |
| +} |
| + |
| +// LoadLayoutIntent reads the log and returns the newest request whose scope |
| +// applies to boxLabel. found=false means either the object is absent, which |
| +// is every deployment's state until an operator records an intent, or it has |
| +// requests but none of them address this box. |
| +// |
| +// A newer schema is an error rather than a best-effort parse: a box that |
| +// cannot be sure what a request means must do nothing, so the request is |
| +// picked up whenever the box is upgraded rather than dropped or guessed at. |
| +func LoadLayoutIntent(ctx context.Context, s Store, boxLabel string) (*LayoutIntent, bool, error) { |
| + key := LayoutIntentKey() |
| + raw, _, err := s.Get(ctx, key) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return nil, false, nil |
| + } |
| + if err != nil { |
| + return nil, false, err |
| + } |
| + g, err := parseLayoutIntents(key, raw) |
| + if err != nil { |
| + return nil, false, err |
| + } |
| + for i := len(g.Requests) - 1; i >= 0; i-- { |
| + if g.Requests[i].Applies(boxLabel) { |
| + in := g.Requests[i] |
| + return &in, true, nil |
| + } |
| + } |
| + return nil, false, nil |
| +} |
| + |
| +// AppendLayoutIntent records one request, oldest-first, after every request |
| +// already in the log. ID and RequestedAt are stamped here when the caller |
| +// leaves them empty, so a caller that only cares about Mode and Scope never |
| +// has to mint either itself. |
| +// |
| +// A CAS conflict is retried, the same shape and for the same reason as |
| +// AppendRestartScope: another writer's request names a different mode or |
| +// scope, so dropping ours would answer "requested" for a convergence that |
| +// will never happen. The retry budget is small on purpose; this is a |
| +// human-paced verb, and a contended object means something is wrong that |
| +// another lap will not fix. |
| +func AppendLayoutIntent(ctx context.Context, s Store, in LayoutIntent) error { |
| + if !ValidLayoutMode(in.Mode) { |
| + return fmt.Errorf("layout intent: mode %q is not flat or nested", in.Mode) |
| + } |
| + if in.ID == "" { |
| + id := make([]byte, 16) |
| + if _, err := io.ReadFull(rand.Reader, id); err != nil { |
| + return err |
| + } |
| + in.ID = hex.EncodeToString(id) |
| + } |
| + if in.RequestedAt == "" { |
| + in.RequestedAt = time.Now().UTC().Format(time.RFC3339) |
| + } |
| + key := LayoutIntentKey() |
| + var lastErr error |
| + for attempt := 0; attempt < 3; attempt++ { |
| + raw, etag, err := s.Get(ctx, key) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + return err |
| + } |
| + g := &LayoutIntents{Schema: 1} |
| + if err == nil { |
| + if g, err = parseLayoutIntents(key, raw); err != nil { |
| + // Refusing beats overwriting: a log this binary cannot read is |
| + // one whose entries it would silently drop on the next write. |
| + return err |
| + } |
| + } else { |
| + etag = "" // create-only |
| + } |
| + g.Schema = 1 |
| + g.Requests = append(g.Requests, in) |
| + body, merr := json.Marshal(g) |
| + if merr != nil { |
| + return merr |
| + } |
| + if _, err := s.Put(ctx, key, body, etag); err != nil { |
| + if errors.Is(err, ErrCASConflict) { |
| + lastErr = err |
| + continue |
| + } |
| + return err |
| + } |
| + return nil |
| + } |
| + return fmt.Errorf("layout intent: %s is being written by someone else; nothing was requested; try again: %w", key, lastErr) |
| +} |
diff --git a/internal/release/layoutintent_test.go b/internal/release/layoutintent_test.go| new file mode 100644 |
| index 000000000..37563ee4d |
| --- /dev/null |
| +++ b/internal/release/layoutintent_test.go |
| @@ -0,0 +1,86 @@ |
| +package release |
| + |
| +// The layout intent's WIRE SHAPE (isolated-exec design §8.2): what lands in |
| +// control/layout.json, what a box makes of it for its own label, and that a |
| +// schema this binary does not understand is refused rather than guessed at. |
| + |
| +import ( |
| + "context" |
| + "testing" |
| +) |
| + |
| +func TestLayoutIntentRoundTripAndScope(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + |
| + if _, found, err := LoadLayoutIntent(ctx, s, "kass-1"); err != nil || found { |
| + t.Fatalf("absent object: found=%v err=%v", found, err) |
| + } |
| + |
| + if err := AppendLayoutIntent(ctx, s, LayoutIntent{Mode: LayoutModeNested, Scope: LayoutScope{Box: "kass-1"}}); err != nil { |
| + t.Fatalf("append: %v", err) |
| + } |
| + |
| + got, found, err := LoadLayoutIntent(ctx, s, "kass-1") |
| + if err != nil || !found || got.Mode != LayoutModeNested { |
| + t.Fatalf("round trip: %+v %v %v", got, found, err) |
| + } |
| + if got.ID == "" || got.RequestedAt == "" { |
| + t.Fatalf("append did not stamp an id and a timestamp: %+v", got) |
| + } |
| + if !got.Applies("kass-1") || got.Applies("kass-2") { |
| + t.Error("box scope must match only the named box") |
| + } |
| + if _, found, err := LoadLayoutIntent(ctx, s, "kass-2"); err != nil || found { |
| + t.Fatalf("a box-scoped intent must not apply to another box: found=%v err=%v", found, err) |
| + } |
| + |
| + // A later fleet-wide request (empty scope) applies everywhere, including |
| + // kass-1, and it is newer, so it is the one that wins there too. |
| + if err := AppendLayoutIntent(ctx, s, LayoutIntent{Mode: LayoutModeFlat}); err != nil { |
| + t.Fatalf("second append: %v", err) |
| + } |
| + got, found, err = LoadLayoutIntent(ctx, s, "kass-1") |
| + if err != nil || !found || got.Mode != LayoutModeFlat { |
| + t.Fatalf("the newest applying request must win: %+v %v %v", got, found, err) |
| + } |
| + if !got.Applies("anything") { |
| + t.Error("an empty scope is the fleet") |
| + } |
| + |
| + // A newer schema is an error, not a guess. The store has no raw put, so |
| + // the fixture is written by deleting the key and putting the body fresh |
| + // (create-only). |
| + if err := s.Delete(ctx, LayoutIntentKey()); err != nil { |
| + t.Fatalf("delete: %v", err) |
| + } |
| + if _, err := s.Put(ctx, LayoutIntentKey(), []byte(`{"schema":2,"requests":[{"mode":"nested"}]}`), ""); err != nil { |
| + t.Fatalf("put newer-schema fixture: %v", err) |
| + } |
| + if _, _, err := LoadLayoutIntent(ctx, s, "kass-1"); err == nil { |
| + t.Fatal("a newer schema must be an error, not a guess") |
| + } |
| +} |
| + |
| +func TestValidLayoutMode(t *testing.T) { |
| + for _, m := range []string{LayoutModeFlat, LayoutModeNested} { |
| + if !ValidLayoutMode(m) { |
| + t.Errorf("ValidLayoutMode(%q) = false, want true", m) |
| + } |
| + } |
| + for _, m := range []string{LayoutModeNone, "bogus"} { |
| + if ValidLayoutMode(m) { |
| + t.Errorf("ValidLayoutMode(%q) = true, want false", m) |
| + } |
| + } |
| +} |
| + |
| +func TestAppendLayoutIntentRefusesAnInvalidMode(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + if err := AppendLayoutIntent(context.Background(), s, LayoutIntent{Mode: "bogus"}); err == nil { |
| + t.Fatal("an invalid mode must be refused, not written") |
| + } |
| + if _, found, err := LoadLayoutIntent(context.Background(), s, "any"); err != nil || found { |
| + t.Fatalf("a refused append must leave the object absent: found=%v err=%v", found, err) |
| + } |
| +} |