amadan

carlos / platform Public

#162: the siblings file — addresses and per-pair keys, no new channel

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
claude 7114b76268c1399eed4bd0c98dfbb6632b6d1f8c parent 7b62b62
6 files changed, +993 −0
  • internal/activator/backend_exec.go +29 −0
  • internal/activator/backend_exec_test.go +52 −0
  • internal/edge/homemap.go +19 −0
  • internal/host/agent.go +31 −0
  • internal/host/siblings.go +463 −0
  • internal/host/siblings_test.go +399 −0
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index 615435540..28f3119bc 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -991,9 +991,38 @@ func (b *ExecBackend) agentOwnedVars(rt registry.Route, fileEnv map[string]strin
} else {
childVars = withoutVars(childVars, "CARLOS_REGION")
}
+ // CARLOS_SIBLINGS_FILE (replica placements design §7): where the agent's
+ // siblings pass wrote this instance's sibling addresses and per-pair
+ // keys. The backend answers it with a stat rather than being handed a
+ // path, because pathsFor is the one place that knows which directory a
+ // row's files live in — /data for a flat row, /data/instances/<id> for a
+ // nested one — and a second copy of that rule in the agent would be a
+ // copy that can drift.
+ //
+ // Agent-owned in both directions, like CARLOS_REGION: stripped when no
+ // file is there, even when a tenant's own env file names one. Lstat and
+ // a regular-file test, not Stat: the directory is tenant-writable, and a
+ // symlink planted under the name is not a file this platform wrote.
+ if sf := filepath.Join(b.pathsFor(rt).dataDir, siblingsFileName); regularFile(sf) {
+ childVars = withVar(childVars, "CARLOS_SIBLINGS_FILE", sf)
+ } else {
+ childVars = withoutVars(childVars, "CARLOS_SIBLINGS_FILE")
+ }
return childVars
}
+// siblingsFileName is the siblings file's name in an instance's data
+// directory. The agent's pass (internal/host/siblings.go) writes it under
+// the same name; this package only ever reads whether it is there.
+const siblingsFileName = "siblings.json"
+
+// regularFile reports whether path is there AND is a plain file — not a
+// symlink, a FIFO or a directory a tenant planted under the name.
+func regularFile(path string) bool {
+ fi, err := os.Lstat(path)
+ return err == nil && fi.Mode().IsRegular()
+}
+
// normalizeSocketMode is the PLATFORM's 0660 on a spawned instance's socket
// (issue #103), not the app's. Extracted from startChild so the unit path
// (whose own bind is the trampoline's, not this process's exec) applies the
diff --git a/internal/activator/backend_exec_test.go b/internal/activator/backend_exec_test.go
index 84f291c12..d154def81 100644
--- a/internal/activator/backend_exec_test.go
+++ b/internal/activator/backend_exec_test.go
@@ -13,6 +13,7 @@ import (
_ "modernc.org/sqlite"
"github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/provision"
"github.com/carlosframework/platform/internal/registry"
)
@@ -107,6 +108,57 @@ func TestExecBackendStartChildRegion(t *testing.T) {
}
}
+// CARLOS_SIBLINGS_FILE (replica placements design §7): the agent's siblings
+// pass writes siblings.json into the instance's own data directory, and the
+// backend — the only side that knows which directory that is — tells the
+// child where it is. Agent-owned in both directions, like CARLOS_REGION
+// above: set when the file is there, STRIPPED when it is not, even when a
+// tenant's own env file tries to name one. The path follows the row's
+// layout: /data for a flat row, /data/instances/<id> for a nested one,
+// which is the only writable /data path inside its mount namespace.
+func TestAgentOwnedVarsSiblingsFile(t *testing.T) {
+ b := testExecBackend(t)
+ const flatHost = "flat.example.com"
+ const nestedHost = "nested.example.com"
+ flatRoute := registry.Route{Host: flatHost, Kind: "instance",
+ DBPath: provision.DefaultDBPath(b.DataDir, flatHost),
+ Socket: filepath.Join(provision.SocketDir, flatHost+".sock")}
+ nestedRoute := registry.Route{Host: nestedHost, Kind: "instance",
+ DBPath: provision.NestedDBPath(b.DataDir, nestedHost),
+ Socket: provision.NestedSocket(nestedHost)}
+ // What a tenant's own env file supplies, which must never survive.
+ tenant := map[string]string{"CARLOS_SIBLINGS_FILE": "/data/forged.json"}
+
+ if got, ok := b.agentOwnedVars(flatRoute, tenant, false)["CARLOS_SIBLINGS_FILE"]; ok {
+ t.Errorf("CARLOS_SIBLINGS_FILE = %q with no file written; want it stripped", got)
+ }
+
+ flatPath := filepath.Join(b.DataDir, "siblings.json")
+ if err := os.WriteFile(flatPath, []byte("{}\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if got := b.agentOwnedVars(flatRoute, tenant, false)["CARLOS_SIBLINGS_FILE"]; got != flatPath {
+ t.Errorf("flat row: CARLOS_SIBLINGS_FILE = %q, want %q", got, flatPath)
+ }
+ // The nested row has its own directory, so the flat file is not its
+ // file and it must still be told nothing.
+ if got, ok := b.agentOwnedVars(nestedRoute, tenant, false)["CARLOS_SIBLINGS_FILE"]; ok {
+ t.Errorf("nested row: CARLOS_SIBLINGS_FILE = %q, want the flat row's file ignored", got)
+ }
+
+ nestedDir := provision.InstanceDir(b.DataDir, nestedHost)
+ if err := os.MkdirAll(nestedDir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ nestedPath := filepath.Join(nestedDir, "siblings.json")
+ if err := os.WriteFile(nestedPath, []byte("{}\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if got := b.agentOwnedVars(nestedRoute, tenant, false)["CARLOS_SIBLINGS_FILE"]; got != nestedPath {
+ t.Errorf("nested row: CARLOS_SIBLINGS_FILE = %q, want %q", got, nestedPath)
+ }
+}
+
// The staging fallback (issue #331): a host whose ordinary prefix is empty
// restores from a move's staged copy — promoted server-side, dropped after
// a clean restore — and the ordinary prefix ALWAYS wins when it exists.
diff --git a/internal/edge/homemap.go b/internal/edge/homemap.go
index 944c8edda..065e17246 100644
--- a/internal/edge/homemap.go
+++ b/internal/edge/homemap.go
@@ -709,6 +709,25 @@ func (m *homeMap) meshKeys() (current, next []byte) {
return m.keyCurrent, m.keyNext
}
+// SiblingEIP is Home for a caller outside this package: which OTHER box
+// homes this host, and where to dial it. The agent's siblings pass
+// (internal/host) needs exactly this to write a replica's dial address into
+// its sibling's file — the address the instance would otherwise have no way
+// to learn — and the refusal to answer for this box's own hosts is right
+// there too, because a pair is never co-located (design §6, §7).
+func (e *Edge) SiblingEIP(host string) (eip, box string, ok bool) {
+ return e.homes.Load().Home(host)
+}
+
+// MeshKeys publishes the mirrored key pair to the agent's siblings pass,
+// which derives a per-pair key from each and never publishes the mesh key
+// itself (design §7). Current first, and next only while a rotation is in
+// flight; both may be nil on a box with nothing mirrored yet, which the pass
+// reads as "not armed" and writes no file for.
+func (e *Edge) MeshKeys() (current, next []byte) {
+ return e.homes.Load().meshKeys()
+}
+
// run polls until stop closes — the edge's standing background-loop shape,
// the same one certMirror.run uses: fixed ticker, errors counted and logged at
// the first and then every tenth, recovery logged once.
diff --git a/internal/host/agent.go b/internal/host/agent.go
index af51537a3..9bb3312d3 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -515,6 +515,13 @@ func RunAgent(args []string) {
// the loop always sees the CURRENT probe set: Reload replaces probers
// behind this handle without the heartbeat needing to be re-wired.
setUpstreamSource(e.UpstreamStates)
+ // The siblings pass's two edge-only inputs (replica placements
+ // design §7): another box's EIP, which only the mirrored home map
+ // knows, and the mesh key pair the per-pair keys are derived from,
+ // which never leaves this process. Method values for
+ // setUpstreamSource's reason — Reload replaces what is behind them
+ // without this needing to be re-wired.
+ setSiblingSource(e.SiblingEIP, e.MeshKeys)
m := edge.NewUsageMeter()
e.EnableUsage(m)
// Same arming point as the meter: the edge observes an upstream 5xx
@@ -2052,6 +2059,17 @@ func provisionTickConfig(getenv func(string) string) provision.ReconcileConfig {
DataDir: dd,
Now: time.Now,
Inbound: inbound,
+ // Every record the pass decoded, in scope or not — the siblings
+ // pass needs both sides of a home/replica pair and only one of them
+ // is ever in this box's scope (replica placements design §7). Set
+ // HERE rather than at a call site because BOTH reconcile legs (the
+ // slow instances loop and the fast converge leg) build their config
+ // from this function, and a pass that ran without the callback
+ // would leave the siblings writer working from a stale map.
+ //
+ // rememberRecords copies before publishing: the map belongs to the
+ // pass, and the callback runs while reconcileMu is held.
+ OnRecords: rememberRecords,
}
// The root release public key, same source and same parser as adoptLoop's
// and domainsTickConfig's. It guards ONE decision in this pass: whether a
@@ -2323,6 +2341,11 @@ func instancesLoop(reg *registry.Registry) {
// predicate, already covered by TestShouldSkipPublisherElection, answers
// both.
isFleetBox := shouldSkipPublisherElection(os.Getenv)
+ // The siblings writer lives across ticks, because the one thing it
+ // carries between them is the derivative it last published per pair —
+ // what covers the window a mesh rotation opens between two boxes
+ // (replica placements design §7).
+ sw := &siblingsWriter{reg: reg, dataDir: cfg.DataDir}
delay := time.Minute
for {
time.Sleep(delay)
@@ -2330,6 +2353,14 @@ func instancesLoop(reg *registry.Registry) {
if err := reconcileSerialized(ctx, store, reg, cfg); err != nil {
log.Printf("agent: instances pass: %v", err)
}
+ // Right after the reconcile whose OnRecords published the record
+ // family, and only once the edge has armed: before that this box
+ // knows no other box's EIP and holds no mesh key.
+ if eipOf, keys, ok := siblingInputs(); ok {
+ if err := sw.pass(latestRecords(), eipOf, keys); err != nil {
+ log.Printf("agent: siblings pass: %v", err)
+ }
+ }
telemetryPass(ctx, store, reg)
runMovesPass(ctx, store)
deploymentRoutes := edgesfile.DeploymentRoutes(ctx, store, time.Now())
diff --git a/internal/host/siblings.go b/internal/host/siblings.go
new file mode 100644
index 000000000..a2a078142
--- /dev/null
+++ b/internal/host/siblings.go
@@ -0,0 +1,463 @@
+// The siblings file (replica placements design §7): the one thing an
+// instance cannot work out for itself about its sibling — where to dial it,
+// and the key to sign the call with. An instance reaches a sibling by
+// dialling that box's EIP with the real hostname as SNI, which is what
+// backhaul does already, so this adds no listener, no private CA and no new
+// platform channel. titogo shapes the lease and drain calls itself.
+package host
+
+import (
+ "bytes"
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "io/fs"
+ "log"
+ "os"
+ "path/filepath"
+ "sort"
+ "sync/atomic"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/instancefs"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// siblingsFileName is the file's name in the instance's data directory, and
+// the only name this package writes or removes there.
+const siblingsFileName = "siblings.json"
+
+// siblingsPort is the port every sibling is dialled on: the box's public
+// edge, presenting the mirrored public certificate for the real hostname
+// (spec §7). Not configurable, for the same reason backhaul's isn't.
+const siblingsPort = "443"
+
+// siblingsFile is one instance's view of its pair, exactly as spec §7 writes
+// it. Role is "home" or "replica" — the side this file is for, never the
+// side it points at.
+type siblingsFile struct {
+ Role string `json:"role"`
+ Host string `json:"host"`
+ Placement string `json:"placement,omitempty"`
+ Home *siblingEntry `json:"home,omitempty"`
+ Replicas []siblingEntry `json:"replicas,omitempty"`
+}
+
+// siblingEntry is the other side of one pair. Keys carries no omitempty on
+// purpose: an entry with an empty list is a fact the reader must see (the
+// pass has no mesh key to derive from), not a field to quietly drop.
+type siblingEntry struct {
+ Host string `json:"host"`
+ Placement string `json:"placement,omitempty"`
+ Addr string `json:"addr"`
+ Keys []string `json:"keys"`
+}
+
+// pairLabel is the HMAC message that makes a key belong to ONE pair:
+// "replica/<account>/<home>/<replica>" (spec §7). Both boxes derive it from
+// the same fleet-wide record map, so both sides of a pair land on the same
+// bytes without ever exchanging them — and a replica compromised in one
+// placement holds a key that verifies only as itself.
+//
+// It doubles as the identity the pass remembers a retired derivative under
+// (siblingsWriter.last), so the two must stay the same string.
+func pairLabel(account, home, replica string) string {
+ return "replica/" + account + "/" + home + "/" + replica
+}
+
+// deriveSiblingKey is the per-pair derivative of one mesh key. HMAC is a PRF,
+// so handing the derivative to a tenant's process exposes nothing about the
+// mesh key itself, which never leaves the agent (mesh.go).
+func deriveSiblingKey(meshKey []byte, label string) string {
+ m := hmac.New(sha256.New, meshKey)
+ m.Write([]byte(label))
+ return base64.StdEncoding.EncodeToString(m.Sum(nil))
+}
+
+// buildSiblings is the pure core: the files this box would write if every
+// host in `all` were its own. Nothing here touches the filesystem and
+// nothing here filters by placement — siblingsWriter.pass writes only the
+// files whose host has a local route, which is the only honest test of "is
+// this instance mine". (Filtering by region would be wrong anyway: a home in
+// the deployment's home region carries Region "".)
+//
+// `all` is the fleet-wide record map ReconcileConfig.OnRecords hands the
+// pass — both sides of a pair are in it even though only one of them is in
+// this box's scope, which is the whole reason the callback exists.
+//
+// eipOf is the edge's home map: the sibling box's EIP, or false for a box
+// that has published no edges file yet. keys is the mesh key list, current
+// first.
+func buildSiblings(all map[string]provision.Record, eipOf func(host string) (string, bool), keys [][]byte) map[string]siblingsFile {
+ out := map[string]siblingsFile{}
+ addr := func(host string) (string, bool) {
+ eip, ok := eipOf(host)
+ if !ok || eip == "" {
+ return "", false
+ }
+ return eip + ":" + siblingsPort, true
+ }
+ derive := func(label string) []string {
+ ks := make([]string, 0, len(keys))
+ for _, k := range keys {
+ if len(k) == 0 {
+ continue
+ }
+ ks = append(ks, deriveSiblingKey(k, label))
+ }
+ return ks
+ }
+
+ // A home that has opted in gets a file even before any replica record
+ // exists, and even when no replica is reachable: "no replicas" and "no
+ // file" must stay different answers (spec §7).
+ for host, rec := range all {
+ if rec.Role == "replica" || len(rec.Replicas) == 0 {
+ continue
+ }
+ out[host] = siblingsFile{Role: "home", Host: host, Placement: rec.Region}
+ }
+
+ for host, rec := range all {
+ if rec.Role != "replica" || rec.Home == "" {
+ continue
+ }
+ homeRec, ok := all[rec.Home]
+ if !ok || homeRec.Role == "replica" {
+ // No home record in the map (or a record pointing at another
+ // replica): nothing to pair with, and a file naming an address
+ // nobody vouched for is worse than no file.
+ continue
+ }
+ label := pairLabel(homeRec.Account, rec.Home, host)
+ ks := derive(label)
+
+ hf, ok := out[rec.Home]
+ if !ok {
+ // The replica record exists but its home's Replicas list does
+ // not name its placement (mid-edit, or a stale record). Pair it
+ // anyway: the replica is serving, and the home needs its
+ // address to drain it.
+ hf = siblingsFile{Role: "home", Host: rec.Home, Placement: homeRec.Region}
+ }
+ if a, ok := addr(host); ok {
+ hf.Replicas = append(hf.Replicas, siblingEntry{
+ Host: host, Placement: rec.Region, Addr: a, Keys: ks,
+ })
+ }
+ out[rec.Home] = hf
+
+ rf := siblingsFile{Role: "replica", Host: host, Placement: rec.Region}
+ if a, ok := addr(rec.Home); ok {
+ rf.Home = &siblingEntry{
+ Host: rec.Home, Placement: homeRec.Region, Addr: a, Keys: ks,
+ }
+ }
+ out[host] = rf
+ }
+
+ // Map iteration order is random and the pass compares the rendered file
+ // against what is on disk to decide whether to write at all, so the
+ // order has to come from the data rather than from the runtime.
+ for host, f := range out {
+ if len(f.Replicas) < 2 {
+ continue
+ }
+ sort.Slice(f.Replicas, func(i, j int) bool { return f.Replicas[i].Host < f.Replicas[j].Host })
+ out[host] = f
+ }
+ return out
+}
+
+// siblingsWriter is the plan's siblingsPass with the one piece of state it
+// cannot recompute: the derivative it last published for each pair.
+//
+// The state is what makes a mesh rotation survivable. meshKeys() answers
+// (current, next) — the pair mirrored DURING a rotation — and after the
+// promotion it answers the new current and nothing else: the old key is gone
+// from the agent entirely. But the two sides of a pair are two files on two
+// boxes rewritten by two passes minutes apart, so for one window each side
+// signs with a key the other has not been told about yet. Republishing the
+// retired derivative for one further pass covers that window, and the
+// receiver accepting ANY entry in the list is what makes the order
+// unimportant (spec §7, revision 3).
+type siblingsWriter struct {
+ reg *registry.Registry
+ dataDir string
+ // last maps a pair's label to the derivative published for it on the
+ // previous pass. Rebuilt every pass from the pairs seen, so a pair that
+ // goes away takes its entry with it rather than accumulating forever.
+ last map[string]string
+}
+
+// pass renders every siblings file this box owns and writes the ones that
+// changed. Called from instancesLoop right after the reconcile pass whose
+// OnRecords handed it `all` — never concurrently with itself.
+func (w *siblingsWriter) pass(all map[string]provision.Record, eipOf func(host string) (string, bool), keys [][]byte) error {
+ if w == nil || w.reg == nil || eipOf == nil {
+ return nil
+ }
+ if len(all) == 0 {
+ // No records at all is a pass that learned nothing — before the
+ // first reconcile has run, or after one that could read nothing.
+ // Treating it as "every pair is gone" would remove every file on
+ // the box on a bucket blip, so it is treated as "not yet" instead.
+ return nil
+ }
+ if len(keys) == 0 {
+ // No mesh key, so no derivative. A file with an empty key list
+ // would read to the app as "armed but unsignable"; an absent file
+ // reads as "not armed yet", which is the truth. Same posture the
+ // edge takes towards an absent mesh key for hop proofs.
+ return nil
+ }
+ built := buildSiblings(all, eipOf, keys)
+ w.republishRetired(built, all)
+
+ routes, err := w.reg.List()
+ if err != nil {
+ return err
+ }
+ // A file belongs to a DIRECTORY, not to a host: the nested layout gives
+ // each instance its own, but every flat row on the box shares /data. So
+ // the decision is made per directory, and that is not tidiness — a
+ // removal keyed by host would delete a flat neighbour's file, and a
+ // write keyed by host would leave two flat rows overwriting each other's
+ // keys turn by turn, with whichever app read last holding a key minted
+ // for someone else's pair.
+ var dirs []string
+ owners := map[string][]string{}
+ for _, rt := range routes {
+ dir := w.dirFor(rt)
+ if _, seen := owners[dir]; !seen {
+ dirs = append(dirs, dir)
+ owners[dir] = nil
+ }
+ if _, ok := built[rt.Host]; ok {
+ owners[dir] = append(owners[dir], rt.Host)
+ }
+ }
+ sort.Strings(dirs)
+
+ var firstErr error
+ note := func(err error) {
+ if err != nil && firstErr == nil {
+ firstErr = err
+ }
+ }
+ for _, dir := range dirs {
+ hosts := owners[dir]
+ sort.Strings(hosts)
+ if len(hosts) > 1 {
+ // Two hosts, one file: refuse both rather than hand one of them
+ // the other's key. Loud, because the fix is an operator's (move
+ // the rows to the isolated layout), not the pass's.
+ log.Printf("agent: siblings: %v share the data directory %s, so none of them gets a %s — a replica pair needs the isolated layout's own directory", hosts, dir, siblingsFileName)
+ hosts = nil
+ }
+ if len(hosts) == 0 {
+ note(removeSiblingsFile(dir))
+ continue
+ }
+ note(writeSiblingsFile(dir, built[hosts[0]]))
+ }
+ return firstErr
+}
+
+// dirFor is the instance's data directory, resolved the way envPathFor
+// resolves the env file (schedule_wiring.go): the SAME activator.Classify
+// the exec backend used to decide where it put the instance's files, so the
+// writer and the reader cannot drift. /data/instances/<id> for a nested row
+// — the only writable /data path inside its mount namespace — and /data for
+// a flat or foreign one.
+func (w *siblingsWriter) dirFor(rt registry.Route) string {
+ if layout, _ := activator.Classify(rt, activator.LayoutInputs{DataDir: w.dataDir}); layout == activator.LayoutNested {
+ return provision.InstanceDir(w.dataDir, rt.Host)
+ }
+ return w.dataDir
+}
+
+// republishRetired folds the previous pass's derivative into each pair's key
+// list and remembers this pass's current one. It mutates built in place: the
+// list the file is rendered from is the list the pair's memory covers.
+func (w *siblingsWriter) republishRetired(built map[string]siblingsFile, all map[string]provision.Record) {
+ next := make(map[string]string, len(w.last))
+ remember := func(label string, ks []string) []string {
+ if len(ks) == 0 {
+ return ks
+ }
+ if prev, ok := w.last[label]; ok && !containsString(ks, prev) {
+ ks = append(ks, prev)
+ }
+ // The CURRENT derivative, never the retired one just appended:
+ // that is what makes the window one pass long rather than forever.
+ next[label] = ks[0]
+ return ks
+ }
+ for host, f := range built {
+ switch f.Role {
+ case "home":
+ for i := range f.Replicas {
+ label := pairLabel(all[host].Account, host, f.Replicas[i].Host)
+ f.Replicas[i].Keys = remember(label, f.Replicas[i].Keys)
+ }
+ case "replica":
+ if f.Home != nil {
+ label := pairLabel(all[f.Home.Host].Account, f.Home.Host, host)
+ f.Home.Keys = remember(label, f.Home.Keys)
+ }
+ }
+ built[host] = f
+ }
+ w.last = next
+}
+
+func containsString(ss []string, s string) bool {
+ for _, v := range ss {
+ if v == s {
+ return true
+ }
+ }
+ return false
+}
+
+// writeSiblingsFile renders f and writes it only if the bytes differ from
+// what is already there — the pass runs every five minutes and a rename per
+// instance per tick, in ground the app is watching, is churn.
+//
+// instancefs.WriteAtomic, never a plain open: the directory is
+// tenant-writable and the file carries a key, so a symlink planted at the
+// name must be displaced by the rename rather than written through. 0600 for
+// the same reason.
+func writeSiblingsFile(dir string, f siblingsFile) error {
+ data, err := json.Marshal(f)
+ if err != nil {
+ return err
+ }
+ data = append(data, '\n')
+ if cur, err := instancefs.ReadBeneathLimit(dir, siblingsFileName, 1<<20); err == nil && bytes.Equal(cur, data) {
+ return nil
+ }
+ if err := instancefs.WriteAtomic(dir, siblingsFileName, data, 0o600); err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ // The instance directory is not there yet (a nested row whose
+ // layout migration has not run). Nothing to report: the next
+ // pass writes it.
+ return nil
+ }
+ return err
+ }
+ return nil
+}
+
+// removeSiblingsFile drops a file whose host is no longer part of any pair.
+//
+// The removal is in the same tenant-writable ground as the write, so it gets
+// the same care. unlink(2) never follows a symlink at the final component,
+// so removing the NAME is safe in itself; what is not safe is removing it
+// through a directory path a tenant could have rerouted, so the path is
+// first resolved with openat2's RESOLVE_NO_SYMLINKS (instancefs.OpenBeneath)
+// and the removal is skipped outright if anything on it is a link.
+func removeSiblingsFile(dir string) error {
+ f, err := instancefs.OpenBeneath(dir, siblingsFileName, os.O_RDONLY, 0)
+ switch {
+ case err == nil:
+ f.Close()
+ case errors.Is(err, fs.ErrNotExist):
+ return nil // the common case: no file, nothing to do
+ case errors.Is(err, instancefs.ErrSymlink), errors.Is(err, instancefs.ErrEscapes):
+ // The name, or a component of the directory, is a link: this path
+ // does not resolve to anything this pass wrote, and unlinking
+ // through it could take a file elsewhere with it. Leave it — the
+ // tenant planted it in their own directory, and the next write
+ // displaces it by rename anyway.
+ return nil
+ case errors.Is(err, instancefs.ErrNotRegular):
+ // A FIFO or a directory planted under the name. Removing it is
+ // still safe (unlink does not follow) and leaves clean ground.
+ default:
+ return err
+ }
+ if err := os.Remove(filepath.Join(dir, siblingsFileName)); err != nil && !errors.Is(err, fs.ErrNotExist) {
+ return err
+ }
+ return nil
+}
+
+// siblingEdge is the live edge's two answers the pass cannot get anywhere
+// else: a box's EIP (the home map's mirror of every other box's edges file)
+// and the mesh key pair. Published by RunAgent's edge-ready callback, read
+// by instancesLoop — the upstreamSource pattern, and for the same reason:
+// the loops start before the edge exists, so the two cannot be wired by
+// passing a handle down.
+type siblingEdge struct {
+ eip func(host string) (eip, box string, ok bool)
+ keys func() (current, next []byte)
+}
+
+var siblingSource atomic.Value // siblingEdge
+
+// setSiblingSource publishes (or, with nils, withdraws) the edge's siblings
+// inputs. Called once, from the edge-ready callback.
+func setSiblingSource(eip func(string) (string, string, bool), keys func() ([]byte, []byte)) {
+ siblingSource.Store(siblingEdge{eip: eip, keys: keys})
+}
+
+// siblingInputs is the pass's view of the published edge: an EIP lookup
+// narrowed to the one question buildSiblings asks, and the mesh keys as a
+// list with the nils dropped, current first.
+//
+// ok is false until the edge-ready callback has fired. That is not merely
+// "not yet": before the edge exists this box knows no other box's EIP and
+// holds no mesh key, and a pass run then would write every home's file with
+// its replicas missing, only to write them back a tick later. No file at all
+// is the honest state.
+func siblingInputs() (eipOf func(host string) (string, bool), keys [][]byte, ok bool) {
+ src, _ := siblingSource.Load().(siblingEdge)
+ if src.eip == nil || src.keys == nil {
+ return nil, nil, false
+ }
+ current, next := src.keys()
+ for _, k := range [][]byte{current, next} {
+ if len(k) > 0 {
+ keys = append(keys, k)
+ }
+ }
+ return func(host string) (string, bool) {
+ eip, _, ok := src.eip(host)
+ return eip, ok
+ }, keys, true
+}
+
+// lastRecords is the record family the most recent reconcile pass saw —
+// EVERY record, including the ones out of this box's scope, which is the
+// half the siblings pass cannot do without: a replica's file names its home,
+// and the home lives on another box.
+//
+// Published by ReconcileConfig.OnRecords (provisionTickConfig sets it, so
+// both reconcile legs carry it) and read by instancesLoop after the pass
+// returns. An atomic.Value rather than a field on the loop because the two
+// legs share one callback and run under reconcileMu.
+var lastRecords atomic.Value // map[string]provision.Record
+
+// rememberRecords is the OnRecords callback. It COPIES: the map belongs to
+// the reconcile pass, the callback runs while reconcileMu is held, and the
+// reader is another goroutine's tick.
+func rememberRecords(all map[string]provision.Record) {
+ cp := make(map[string]provision.Record, len(all))
+ for host, rec := range all {
+ cp[host] = rec
+ }
+ lastRecords.Store(cp)
+}
+
+// latestRecords is the last pass's record family, or nil before any pass has
+// run — which the siblings pass reads as "nothing to write yet", never as
+// "every pair is gone".
+func latestRecords() map[string]provision.Record {
+ all, _ := lastRecords.Load().(map[string]provision.Record)
+ return all
+}
diff --git a/internal/host/siblings_test.go b/internal/host/siblings_test.go
new file mode 100644
index 000000000..6c50afe1a
--- /dev/null
+++ b/internal/host/siblings_test.go
@@ -0,0 +1,399 @@
+package host
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "reflect"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// The pair a home and its replica make: each side's file names the other,
+// with the other's dial address and the key of THIS pair (spec §7). The two
+// files are written by two different boxes, from the same fleet-wide record
+// map, so the derivation must land on the same bytes on both sides — that
+// last assertion is the whole protocol.
+func TestBuildSiblingsPairsHomeAndReplica(t *testing.T) {
+ all := map[string]provision.Record{
+ "acme.go.tito.io": {Host: "acme.go.tito.io", Account: "bab", App: "tito", Region: "ie",
+ Steering: "latency", Replicas: []string{"uk"}},
+ "r7x2.go.tito.io": {Host: "r7x2.go.tito.io", Account: "bab", App: "tito", Region: "uk",
+ Role: "replica", Home: "acme.go.tito.io"},
+ }
+ eip := func(host string) (string, bool) {
+ return map[string]string{"acme.go.tito.io": "198.51.100.4", "r7x2.go.tito.io": "203.0.113.9"}[host], true
+ }
+ got := buildSiblings(all, eip, [][]byte{[]byte("mesh-current")})
+
+ home := got["acme.go.tito.io"]
+ if home.Role != "home" || len(home.Replicas) != 1 {
+ t.Fatalf("home file = %+v, want one replica", home)
+ }
+ if home.Replicas[0].Addr != "203.0.113.9:443" || home.Replicas[0].Placement != "uk" {
+ t.Errorf("replica entry = %+v", home.Replicas[0])
+ }
+ replica := got["r7x2.go.tito.io"]
+ if replica.Role != "replica" || replica.Home == nil || replica.Home.Addr != "198.51.100.4:443" {
+ t.Fatalf("replica file = %+v", replica)
+ }
+ if len(home.Replicas[0].Keys) != 1 || home.Replicas[0].Keys[0] != replica.Home.Keys[0] {
+ t.Error("the two sides of a pair must carry the same key")
+ }
+}
+
+// Per PAIR, not per mesh key: a replica compromised in one placement can
+// speak as itself and never as its sibling (spec §7). And a rotation in
+// flight publishes both derivatives, current first, because the two sides'
+// files are rewritten by two passes minutes apart.
+func TestBuildSiblingsKeysArePerPairAndRotate(t *testing.T) {
+ all := map[string]provision.Record{
+ "acme.go.tito.io": {Host: "acme.go.tito.io", Account: "bab", App: "tito", Region: "ie", Steering: "latency", Replicas: []string{"uk", "de"}},
+ "r7x2.go.tito.io": {Host: "r7x2.go.tito.io", Account: "bab", App: "tito", Region: "uk", Role: "replica", Home: "acme.go.tito.io"},
+ "r8y3.go.tito.io": {Host: "r8y3.go.tito.io", Account: "bab", App: "tito", Region: "de", Role: "replica", Home: "acme.go.tito.io"},
+ }
+ eip := func(host string) (string, bool) { return "203.0.113.9", true }
+ got := buildSiblings(all, eip, [][]byte{[]byte("cur"), []byte("prev")})
+ home := got["acme.go.tito.io"]
+ if len(home.Replicas) != 2 {
+ t.Fatalf("want two replicas, got %+v", home.Replicas)
+ }
+ if home.Replicas[0].Keys[0] == home.Replicas[1].Keys[0] {
+ t.Error("two replicas of one home share a key: a compromised replica could speak as its sibling")
+ }
+ if len(home.Replicas[0].Keys) != 2 {
+ t.Error("a rotation in flight must publish both keys, current first")
+ }
+}
+
+// A box that has published no edges file yet has no EIP, and an entry with
+// an empty addr would be worse than no entry — so it is omitted. The home's
+// own file is still written, so the app can tell "no replicas" from "no
+// file" (spec §7).
+func TestBuildSiblingsOmitsASiblingWithNoEIPAndStillWritesAnEmptyHome(t *testing.T) {
+ all := map[string]provision.Record{
+ "acme.go.tito.io": {Host: "acme.go.tito.io", Account: "bab", App: "tito", Region: "ie", Steering: "latency", Replicas: []string{"uk"}},
+ "r7x2.go.tito.io": {Host: "r7x2.go.tito.io", Account: "bab", App: "tito", Region: "uk", Role: "replica", Home: "acme.go.tito.io"},
+ }
+ got := buildSiblings(all, func(string) (string, bool) { return "", false }, [][]byte{[]byte("cur")})
+ home, ok := got["acme.go.tito.io"]
+ if !ok {
+ t.Fatal("a home with no reachable replica still gets a file, so the app can tell 'no replicas' from 'no file'")
+ }
+ if len(home.Replicas) != 0 {
+ t.Errorf("a sibling with no EIP must be omitted, not written with an empty addr: %+v", home.Replicas)
+ }
+}
+
+// An account with no replicas is the fleet's whole population today: it must
+// produce no file at all, not an empty one (global constraint: additive
+// only).
+func TestBuildSiblingsIgnoresAnAccountWithNoReplicas(t *testing.T) {
+ all := map[string]provision.Record{
+ "plain.go.tito.io": {Host: "plain.go.tito.io", Account: "zed", App: "tito", Region: "ie"},
+ }
+ if got := buildSiblings(all, func(string) (string, bool) { return "203.0.113.9", true }, [][]byte{[]byte("cur")}); len(got) != 0 {
+ t.Errorf("buildSiblings = %+v, want nothing for an account with no replicas", got)
+ }
+}
+
+// --- the writer ---
+
+const (
+ siblingsHomeHost = "acme.go.tito.io"
+ siblingsReplicaHost = "r7x2.go.tito.io"
+)
+
+// siblingsPair is the two records every writer test below works from: a home
+// opted into one replica placement, and the replica record itself.
+func siblingsPair() map[string]provision.Record {
+ return map[string]provision.Record{
+ siblingsHomeHost: {Host: siblingsHomeHost, Account: "bab", App: "tito", Region: "ie",
+ Steering: "latency", Replicas: []string{"uk"}},
+ siblingsReplicaHost: {Host: siblingsReplicaHost, Account: "bab", App: "tito", Region: "uk",
+ Role: "replica", Home: siblingsHomeHost},
+ }
+}
+
+func siblingsEIP(host string) (string, bool) {
+ eip := map[string]string{siblingsHomeHost: "198.51.100.4", siblingsReplicaHost: "203.0.113.9"}[host]
+ return eip, eip != ""
+}
+
+// addFlatRoute and addNestedRoute give the registry a row whose DBPath and
+// Socket are exactly one layout's defaults, which is what activator.Classify
+// reads — the same classification the exec backend used to decide where it
+// put the instance's files in the first place.
+func addFlatRoute(t *testing.T, reg *registry.Registry, dd, host string) {
+ t.Helper()
+ if err := reg.Upsert(registry.Route{
+ Host: host, Kind: "instance", Account: "bab", App: "tito", Status: "ready", Hibernate: true,
+ DBPath: provision.DefaultDBPath(dd, host), Socket: filepath.Join(provision.SocketDir, host+".sock"),
+ }); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func addNestedRoute(t *testing.T, reg *registry.Registry, dd, host string) {
+ t.Helper()
+ if err := os.MkdirAll(provision.InstanceDir(dd, host), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.Upsert(registry.Route{
+ Host: host, Kind: "instance", Account: "bab", App: "tito", Status: "ready", Hibernate: true,
+ DBPath: provision.NestedDBPath(dd, host), Socket: provision.NestedSocket(host),
+ }); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func openSiblingsRegistry(t *testing.T, dd string) *registry.Registry {
+ t.Helper()
+ reg, err := registry.Open(filepath.Join(dd, "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { reg.Close() })
+ return reg
+}
+
+func readSiblingsFile(t *testing.T, path string) siblingsFile {
+ t.Helper()
+ raw, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("reading %s: %v", path, err)
+ }
+ var f siblingsFile
+ if err := json.Unmarshal(raw, &f); err != nil {
+ t.Fatalf("%s is not the file spec §7 describes: %v\n%s", path, err, raw)
+ }
+ return f
+}
+
+// The pass writes only the files whose host has a route on THIS box, and it
+// writes each one where that host's own layout puts its files: /data for a
+// flat row, /data/instances/<id> for a nested one — the only writable /data
+// path inside an isolated instance's mount namespace. Mode 0600: the
+// directory is tenant-writable ground and the key is a secret.
+func TestSiblingsPassWritesEachLayoutsDirectoryAt0600(t *testing.T) {
+ dd := t.TempDir()
+ reg := openSiblingsRegistry(t, dd)
+ addFlatRoute(t, reg, dd, siblingsHomeHost)
+ addNestedRoute(t, reg, dd, siblingsReplicaHost)
+
+ all := siblingsPair()
+ // A third record, homed on some other box: in the map, not in the
+ // registry, so no file of any kind lands here for it.
+ all["other.go.tito.io"] = provision.Record{Host: "other.go.tito.io", Account: "zed", App: "tito",
+ Region: "de", Steering: "latency", Replicas: []string{"fr"}}
+
+ w := &siblingsWriter{reg: reg, dataDir: dd}
+ if err := w.pass(all, siblingsEIP, [][]byte{[]byte("mesh-A")}); err != nil {
+ t.Fatalf("pass: %v", err)
+ }
+
+ homePath := filepath.Join(dd, siblingsFileName)
+ home := readSiblingsFile(t, homePath)
+ if home.Role != "home" || len(home.Replicas) != 1 || home.Replicas[0].Addr != "203.0.113.9:443" {
+ t.Errorf("flat home file = %+v", home)
+ }
+ fi, err := os.Stat(homePath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fi.Mode().Perm() != 0o600 {
+ t.Errorf("%s mode = %v, want 0600: the file carries a key", homePath, fi.Mode().Perm())
+ }
+
+ nestedPath := filepath.Join(provision.InstanceDir(dd, siblingsReplicaHost), siblingsFileName)
+ replica := readSiblingsFile(t, nestedPath)
+ if replica.Role != "replica" || replica.Home == nil || replica.Home.Addr != "198.51.100.4:443" {
+ t.Errorf("nested replica file = %+v", replica)
+ }
+ if _, err := os.Stat(filepath.Join(dd, siblingsReplicaHost+".env")); err == nil {
+ t.Error("a nested row's file must not be written flat as well")
+ }
+ if _, err := os.Stat(filepath.Join(provision.InstanceDir(dd, "other.go.tito.io"), siblingsFileName)); err == nil {
+ t.Error("a file was written for a host with no route on this box")
+ }
+}
+
+// A pass that would write the same bytes writes nothing at all: the file is
+// re-rendered every five minutes and a rename per instance per tick, in
+// tenant ground, is churn the app would see as a change.
+func TestSiblingsPassRewritesNothingWhenThePairIsUnchanged(t *testing.T) {
+ dd := t.TempDir()
+ reg := openSiblingsRegistry(t, dd)
+ addFlatRoute(t, reg, dd, siblingsHomeHost)
+
+ w := &siblingsWriter{reg: reg, dataDir: dd}
+ keys := [][]byte{[]byte("mesh-A")}
+ if err := w.pass(siblingsPair(), siblingsEIP, keys); err != nil {
+ t.Fatalf("first pass: %v", err)
+ }
+ path := filepath.Join(dd, siblingsFileName)
+ before, err := os.Stat(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := w.pass(siblingsPair(), siblingsEIP, keys); err != nil {
+ t.Fatalf("second pass: %v", err)
+ }
+ after, err := os.Stat(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !os.SameFile(before, after) {
+ t.Error("an unchanged pair was rewritten: WriteAtomic renamed a new file over the old one")
+ }
+}
+
+// A host that stops being part of any pair loses its file — and the removal
+// happens in tenant-writable ground, so it must displace a planted symlink
+// rather than follow it.
+func TestSiblingsPassRemovesAStaleFileWithoutFollowingASymlink(t *testing.T) {
+ dd := t.TempDir()
+ reg := openSiblingsRegistry(t, dd)
+ addFlatRoute(t, reg, dd, siblingsHomeHost)
+
+ w := &siblingsWriter{reg: reg, dataDir: dd}
+ if err := w.pass(siblingsPair(), siblingsEIP, [][]byte{[]byte("mesh-A")}); err != nil {
+ t.Fatalf("first pass: %v", err)
+ }
+ path := filepath.Join(dd, siblingsFileName)
+ if _, err := os.Stat(path); err != nil {
+ t.Fatalf("first pass wrote no file: %v", err)
+ }
+
+ // The account's replicas are withdrawn: the record no longer names a
+ // placement and the replica record is gone.
+ plain := map[string]provision.Record{
+ siblingsHomeHost: {Host: siblingsHomeHost, Account: "bab", App: "tito", Region: "ie"},
+ }
+ if err := w.pass(plain, siblingsEIP, [][]byte{[]byte("mesh-A")}); err != nil {
+ t.Fatalf("second pass: %v", err)
+ }
+ if _, err := os.Lstat(path); !os.IsNotExist(err) {
+ t.Errorf("a stale siblings.json survived the pass that stopped writing it, lstat err = %v", err)
+ }
+
+ // Now the attack: a tenant plants a symlink at the file's name pointing
+ // at something outside its directory, and the pass removes it again.
+ canary := filepath.Join(t.TempDir(), "canary")
+ if err := os.WriteFile(canary, []byte("not yours"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(canary, path); err != nil {
+ t.Fatal(err)
+ }
+ if err := w.pass(plain, siblingsEIP, [][]byte{[]byte("mesh-A")}); err != nil {
+ t.Fatalf("third pass: %v", err)
+ }
+ if _, err := os.Stat(canary); err != nil {
+ t.Fatalf("the stale-file removal followed a planted symlink and deleted a file outside the instance directory: %v", err)
+ }
+}
+
+// The flat layout puts every row's files in one shared /data, so two flat
+// hosts that both want a siblings file want the SAME file. Neither gets one,
+// and an existing one is removed: an app handed a file minted for its
+// neighbour's pair would sign a lease call with a key that belongs to
+// another account. The isolated layout's per-instance directory is the fix,
+// and only an operator can apply it, so the pass says so.
+func TestSiblingsPassRefusesTwoFlatHostsSharingOneDirectory(t *testing.T) {
+ dd := t.TempDir()
+ reg := openSiblingsRegistry(t, dd)
+ addFlatRoute(t, reg, dd, siblingsHomeHost)
+ w := &siblingsWriter{reg: reg, dataDir: dd}
+ if err := w.pass(siblingsPair(), siblingsEIP, [][]byte{[]byte("mesh-A")}); err != nil {
+ t.Fatalf("first pass: %v", err)
+ }
+ path := filepath.Join(dd, siblingsFileName)
+ if _, err := os.Stat(path); err != nil {
+ t.Fatalf("the single flat host got no file: %v", err)
+ }
+
+ // A second flat host on the same box, with replicas of its own.
+ const other = "zed.go.tito.io"
+ addFlatRoute(t, reg, dd, other)
+ all := siblingsPair()
+ all[other] = provision.Record{Host: other, Account: "zed", App: "tito", Region: "ie",
+ Steering: "latency", Replicas: []string{"uk"}}
+ if err := w.pass(all, siblingsEIP, [][]byte{[]byte("mesh-A")}); err != nil {
+ t.Fatalf("second pass: %v", err)
+ }
+ if _, err := os.Lstat(path); !os.IsNotExist(err) {
+ f := readSiblingsFile(t, path)
+ t.Errorf("one of two flat hosts kept the shared file (%+v); neither may have it", f)
+ }
+}
+
+// Rotation (spec §7, revision 3). meshKeys() hands the pass (current, next),
+// and after a promotion the old key is simply GONE — so the pass keeps its
+// own memory: the derivative it last wrote for a pair is republished for one
+// further pass, and then retired. Without it, the two sides of a pair — two
+// files on two boxes, rewritten by two passes minutes apart — would spend
+// that window signing with keys the other end had never heard of.
+func TestSiblingsPassRepublishesTheRetiredKeyForOnePass(t *testing.T) {
+ dd := t.TempDir()
+ reg := openSiblingsRegistry(t, dd)
+ addFlatRoute(t, reg, dd, siblingsHomeHost)
+ w := &siblingsWriter{reg: reg, dataDir: dd}
+ path := filepath.Join(dd, siblingsFileName)
+
+ if err := w.pass(siblingsPair(), siblingsEIP, [][]byte{[]byte("mesh-A")}); err != nil {
+ t.Fatalf("pass 1: %v", err)
+ }
+ fromA := readSiblingsFile(t, path).Replicas[0].Keys
+ if len(fromA) != 1 {
+ t.Fatalf("pass 1 keys = %v, want just the derivative of mesh-A", fromA)
+ }
+
+ // The rotation promotes: meshKeys() now answers with B alone.
+ if err := w.pass(siblingsPair(), siblingsEIP, [][]byte{[]byte("mesh-B")}); err != nil {
+ t.Fatalf("pass 2: %v", err)
+ }
+ fromB := readSiblingsFile(t, path).Replicas[0].Keys
+ if len(fromB) != 2 || fromB[1] != fromA[0] {
+ t.Fatalf("pass 2 keys = %v, want the new derivative first and the retired one still published", fromB)
+ }
+ if fromB[0] == fromA[0] {
+ t.Fatal("pass 2 published the old derivative as current")
+ }
+
+ // One further pass and the retired derivative is gone.
+ if err := w.pass(siblingsPair(), siblingsEIP, [][]byte{[]byte("mesh-B")}); err != nil {
+ t.Fatalf("pass 3: %v", err)
+ }
+ third := readSiblingsFile(t, path).Replicas[0].Keys
+ if !reflect.DeepEqual(third, []string{fromB[0]}) {
+ t.Errorf("pass 3 keys = %v, want only the current derivative", third)
+ }
+}
+
+// The memory is per PAIR and bounded: a pair that disappears takes its
+// remembered derivative with it, or a box that has minted and retired
+// replicas for a year would carry every one of them forever.
+func TestSiblingsPassForgetsAPairThatIsGone(t *testing.T) {
+ dd := t.TempDir()
+ reg := openSiblingsRegistry(t, dd)
+ addFlatRoute(t, reg, dd, siblingsHomeHost)
+ w := &siblingsWriter{reg: reg, dataDir: dd}
+
+ if err := w.pass(siblingsPair(), siblingsEIP, [][]byte{[]byte("mesh-A")}); err != nil {
+ t.Fatalf("pass 1: %v", err)
+ }
+ if len(w.last) != 1 {
+ t.Fatalf("remembered %d derivatives, want one per pair", len(w.last))
+ }
+ if err := w.pass(map[string]provision.Record{
+ siblingsHomeHost: {Host: siblingsHomeHost, Account: "bab", App: "tito", Region: "ie"},
+ }, siblingsEIP, [][]byte{[]byte("mesh-A")}); err != nil {
+ t.Fatalf("pass 2: %v", err)
+ }
+ if len(w.last) != 0 {
+ t.Errorf("the pass still remembers %v after the pair was withdrawn", w.last)
+ }
+}