amadan

carlos / platform Public

edge: two boxes claiming one host are settled by the newer installation's stamp

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude e9246bda72c6f07e79ce96056d4adc93a2ed9201 parent 2ca53cd
4 files changed, +223 −6
  • internal/edge/homemap.go +84 −6
  • internal/edge/homemap_test.go +123 −0
  • internal/host/edgesfile.go +5 −0
  • internal/host/edgesfile_test.go +11 −0
diff --git a/internal/edge/homemap.go b/internal/edge/homemap.go
index d51daa8be..6558dbd8b 100644
--- a/internal/edge/homemap.go
+++ b/internal/edge/homemap.go
@@ -155,6 +155,8 @@ type edgesHostView struct {
// them. The edge-local door refuses a remote host with no owner.
Account string `json:"account,omitempty"`
App string `json:"app,omitempty"`
+ // Stamp orders two boxes' claims on one host: see foldHomes.
+ Stamp int64 `json:"stamp,omitempty"`
}
// homeEntry is one folded answer: where to dial, and which box that is.
@@ -173,6 +175,8 @@ type homeEntry struct {
// account and app are the owner the dial target's box published. They
// travel with eip and box and are never merged across entries.
account, app string
+ // stamp is the dial winner's installation stamp, 0 when unstamped.
+ stamp int64
}
// homeSnapshot is one poll's result, swapped in whole and immutable once
@@ -326,7 +330,14 @@ func (m *homeMap) refreshHosts(ctx context.Context) error {
if failed > 0 {
m.logPartialPoll(failed, tried, lastErr)
}
- snap := &homeSnapshot{files: next, hosts: foldHomes(next, m.self)}
+ hosts, tied := foldHomes(next, m.self)
+ if len(tied) > 0 {
+ // One claim revision names one executor, so two boxes publishing
+ // one host at the same stamp is a fault upstream. The fold stays
+ // deterministic; say so once a refresh.
+ log.Printf("edge: home map: %d host(s) published by two boxes at the same installation stamp, e.g. %s", len(tied), tied[0])
+ }
+ snap := &homeSnapshot{files: next, hosts: hosts}
m.mu.Lock()
m.snap = snap
wake := m.hostsWake
@@ -535,7 +546,7 @@ func (m *homeMap) logPartialPoll(failed, tried int, err error) {
// split-and-wrong: the whole fleet aiming at one box is a condition that heals
// on the next publish, where half the fleet aiming at each is a condition that
// looks fine from any single box.
-func foldHomes(files map[string]edgesFileView, self string) map[string]homeEntry {
+func foldHomes(files map[string]edgesFileView, self string) (map[string]homeEntry, []string) {
keys := make([]string, 0, len(files))
for k := range files {
keys = append(keys, k)
@@ -573,7 +584,7 @@ func foldHomes(files map[string]edgesFileView, self string) map[string]homeEntry
continue
}
in := homeEntry{eip: f.EIP, box: f.Box, platform: h.Platform && !h.Alias && !h.Wildcard, alias: h.Alias, wildcard: h.Wildcard, dial: h.Role == edgeRoleHome,
- account: h.Account, app: h.App}
+ account: h.Account, app: h.App, stamp: h.Stamp}
prev, dup := hosts[host]
if !dup {
hosts[host] = in
@@ -591,14 +602,81 @@ func foldHomes(files map[string]edgesFileView, self string) map[string]homeEntry
merged := prev
merged.platform = prev.platform || in.platform
merged.wildcard = prev.wildcard || in.wildcard
- if in.dial && (!prev.dial || (prev.alias && !in.alias)) {
+ if dialBeats(in, prev) {
merged.eip, merged.box, merged.alias, merged.dial = in.eip, in.box, in.alias, true
- merged.account, merged.app = in.account, in.app
+ merged.account, merged.app, merged.stamp = in.account, in.app, in.stamp
}
hosts[host] = merged
}
}
- return hosts
+ return hosts, tiedStamps(files)
+}
+
+// tiedStamps lists, sorted, the hosts that two boxes publish as home at
+// the same positive stamp. It looks at every file, this box's own
+// included, and at every pair, not only at whichever entry won the dial:
+// in a two-box region each box must be able to see that it and its
+// neighbour disagree, and a third box's higher stamp must not hide a tie
+// between two others.
+//
+// Unstamped duplicates are not reported. They are normal: a static site
+// or a public address route runs on every box, and every box publishes it.
+func tiedStamps(files map[string]edgesFileView) []string {
+ type key struct {
+ host string
+ stamp int64
+ }
+ boxes := map[key]map[string]bool{}
+ for _, f := range files {
+ if f.Box == "" {
+ continue
+ }
+ for host, h := range f.Hosts {
+ if host == "" || h.Role != edgeRoleHome || h.Stamp <= 0 {
+ continue
+ }
+ k := key{host, h.Stamp}
+ if boxes[k] == nil {
+ boxes[k] = map[string]bool{}
+ }
+ boxes[k][f.Box] = true
+ }
+ }
+ seen := map[string]bool{}
+ var tied []string
+ for k, set := range boxes {
+ if len(set) > 1 && !seen[k.host] {
+ seen[k.host] = true
+ tied = append(tied, k.host)
+ }
+ }
+ slices.Sort(tied)
+ return tied
+}
+
+// dialBeats reports whether a later file's entry takes the dial from the
+// one already folded. In order:
+//
+// - only a dial target can take it, and any dial target beats none;
+// - a ROUTE beats an ALIAS whatever else is true (#287);
+// - between two of the same kind, the newer installation wins: a stamped
+// entry beats an unstamped one, and a higher stamp beats a lower one
+// (region HA spec 1 HA4). A dead box's file lingers; its successor's
+// stamp is always higher, so the stale one loses without any liveness
+// rule;
+// - otherwise the earlier file keeps it, which is the first box in
+// sorted order, as before.
+func dialBeats(in, prev homeEntry) bool {
+ if !in.dial {
+ return false
+ }
+ if !prev.dial {
+ return true
+ }
+ if prev.alias != in.alias {
+ return prev.alias && !in.alias
+ }
+ return in.stamp > prev.stamp
}
func (m *homeMap) snapshot() *homeSnapshot {
diff --git a/internal/edge/homemap_test.go b/internal/edge/homemap_test.go
index fc3594a2d..d5fe3676a 100644
--- a/internal/edge/homemap_test.go
+++ b/internal/edge/homemap_test.go
@@ -1633,3 +1633,126 @@ func TestOwnerOfARouteReplacingAnAliasIsTheRoutes(t *testing.T) {
t.Fatalf("Home(ownerless.test) = %s, want carol", box)
}
}
+
+func TestHomeMapHigherStampWinsWhateverTheSortOrder(t *testing.T) {
+ s := meshTestStore(t)
+ // "annie" sorts first and is the dead box; "betty" holds the newer installation.
+ putJSON(t, s, "control/edges/annie.json", `{"v":1,"box":"annie","eip":"1.1.1.1","hosts":{"shop.test":{"role":"home","stamp":41}}}`)
+ putJSON(t, s, "control/edges/betty.json", `{"v":1,"box":"betty","eip":"2.2.2.2","hosts":{"shop.test":{"role":"home","stamp":57}}}`)
+ m := newHomeMap(s, "carol")
+ refreshHomeMap(t, m)
+ if eip, box, ok := m.Home("shop.test"); !ok || box != "betty" || eip != "2.2.2.2" {
+ t.Fatalf("Home = %q %q %v, want betty", eip, box, ok)
+ }
+ // And the other way round, so it is the stamp and not the name.
+ putJSON(t, s, "control/edges/annie.json", `{"v":1,"box":"annie","eip":"1.1.1.1","hosts":{"shop.test":{"role":"home","stamp":90}}}`)
+ refreshHomeMap(t, m)
+ if _, box, _ := m.Home("shop.test"); box != "annie" {
+ t.Fatalf("Home box = %q, want annie", box)
+ }
+}
+
+func TestHomeMapStampedBeatsUnstamped(t *testing.T) {
+ s := meshTestStore(t)
+ putJSON(t, s, "control/edges/annie.json", `{"v":1,"box":"annie","eip":"1.1.1.1","hosts":{"shop.test":{"role":"home"}}}`)
+ putJSON(t, s, "control/edges/betty.json", `{"v":1,"box":"betty","eip":"2.2.2.2","hosts":{"shop.test":{"role":"home","stamp":3}}}`)
+ m := newHomeMap(s, "carol")
+ refreshHomeMap(t, m)
+ if _, box, _ := m.Home("shop.test"); box != "betty" {
+ t.Fatalf("Home box = %q, want betty", box)
+ }
+}
+
+func TestHomeMapUnstampedKeepsTodaysRule(t *testing.T) {
+ s := meshTestStore(t)
+ putJSON(t, s, "control/edges/annie.json", `{"v":1,"box":"annie","eip":"1.1.1.1","hosts":{"shop.test":{"role":"home","alias":true}}}`)
+ putJSON(t, s, "control/edges/betty.json", `{"v":1,"box":"betty","eip":"2.2.2.2","hosts":{"shop.test":{"role":"home"}}}`)
+ m := newHomeMap(s, "carol")
+ refreshHomeMap(t, m)
+ if _, box, _ := m.Home("shop.test"); box != "betty" {
+ t.Fatalf("a route still beats an alias: got %q", box)
+ }
+ putJSON(t, s, "control/edges/annie.json", `{"v":1,"box":"annie","eip":"1.1.1.1","hosts":{"shop.test":{"role":"home"}}}`)
+ refreshHomeMap(t, m)
+ if _, box, _ := m.Home("shop.test"); box != "annie" {
+ t.Fatalf("two plain routes: first box in sorted order; got %q", box)
+ }
+}
+
+func TestHomeMapAStampedAliasDoesNotBeatAnUnstampedRoute(t *testing.T) {
+ s := meshTestStore(t)
+ putJSON(t, s, "control/edges/annie.json", `{"v":1,"box":"annie","eip":"1.1.1.1","hosts":{"shop.test":{"role":"home"}}}`)
+ putJSON(t, s, "control/edges/betty.json", `{"v":1,"box":"betty","eip":"2.2.2.2","hosts":{"shop.test":{"role":"home","alias":true,"stamp":9}}}`)
+ m := newHomeMap(s, "carol")
+ refreshHomeMap(t, m)
+ if _, box, _ := m.Home("shop.test"); box != "annie" {
+ t.Fatalf("got %q: #287's rule (a route beats an alias) comes before stamps", box)
+ }
+}
+
+func TestHomeMapAStampedEntryWithNoDialAddressDoesNotWin(t *testing.T) {
+ s := meshTestStore(t)
+ putJSON(t, s, "control/edges/annie.json", `{"v":1,"box":"annie","eip":"1.1.1.1","hosts":{"shop.test":{"role":"home","stamp":2}}}`)
+ putJSON(t, s, "control/edges/betty.json", `{"v":1,"box":"betty","eip":"","hosts":{"shop.test":{"role":"home","stamp":99}}}`)
+ m := newHomeMap(s, "carol")
+ refreshHomeMap(t, m)
+ if eip, box, ok := m.Home("shop.test"); !ok || box != "annie" || eip != "1.1.1.1" {
+ t.Fatalf("Home = %q %q %v, want annie: a box nothing can dial is skipped today and stays skipped", eip, box, ok)
+ }
+}
+
+func file(box, eip string, hosts map[string]edgesHostView) edgesFileView {
+ return edgesFileView{V: 1, Box: box, EIP: eip, Hosts: hosts}
+}
+
+func TestHomeMapReportsTwoBoxesAtTheSameStamp(t *testing.T) {
+ files := map[string]edgesFileView{
+ "control/edges/annie.json": file("annie", "1.1.1.1", map[string]edgesHostView{
+ "tied.test": {Role: "home", Stamp: 7}, "fine.test": {Role: "home", Stamp: 7}, "plain.test": {Role: "home"}}),
+ "control/edges/betty.json": file("betty", "2.2.2.2", map[string]edgesHostView{
+ "tied.test": {Role: "home", Stamp: 7}, "fine.test": {Role: "home", Stamp: 9}, "plain.test": {Role: "home"}}),
+ }
+ hosts, tied := foldHomes(files, "carol")
+ if len(tied) != 1 || tied[0] != "tied.test" {
+ t.Fatalf("tied = %v, want [tied.test]: unstamped duplicates are normal (static sites run on every box) and are not reported", tied)
+ }
+ if hosts["tied.test"].box != "annie" {
+ t.Fatalf("a tie keeps the first box in sorted order; got %q", hosts["tied.test"].box)
+ }
+ if hosts["fine.test"].box != "betty" {
+ t.Fatalf("fine.test box = %q", hosts["fine.test"].box)
+ }
+}
+
+// A tie between two boxes is reported even when a third box holds a higher
+// stamp and wins the dial: the report does not depend on the winner.
+func TestHomeMapReportsATieBelowTheWinner(t *testing.T) {
+ files := map[string]edgesFileView{
+ "control/edges/annie.json": file("annie", "1.1.1.1", map[string]edgesHostView{"h.test": {Role: "home", Stamp: 9}}),
+ "control/edges/betty.json": file("betty", "2.2.2.2", map[string]edgesHostView{"h.test": {Role: "home", Stamp: 7}}),
+ "control/edges/carol.json": file("carol", "3.3.3.3", map[string]edgesHostView{"h.test": {Role: "home", Stamp: 7}}),
+ }
+ hosts, tied := foldHomes(files, "dave")
+ if len(tied) != 1 || tied[0] != "h.test" {
+ t.Fatalf("tied = %v, want [h.test]", tied)
+ }
+ if hosts["h.test"].box != "annie" {
+ t.Fatalf("the highest stamp still wins the dial; got %q", hosts["h.test"].box)
+ }
+}
+
+// In a two-box region each box excludes itself from routing. It must
+// still see that it and its neighbour publish one host at one stamp.
+func TestHomeMapReportsATieWithThisBoxItself(t *testing.T) {
+ files := map[string]edgesFileView{
+ "control/edges/annie.json": file("annie", "1.1.1.1", map[string]edgesHostView{"h.test": {Role: "home", Stamp: 7}}),
+ "control/edges/betty.json": file("betty", "2.2.2.2", map[string]edgesHostView{"h.test": {Role: "home", Stamp: 7}}),
+ }
+ hosts, tied := foldHomes(files, "annie")
+ if len(tied) != 1 || tied[0] != "h.test" {
+ t.Fatalf("tied = %v, want [h.test]", tied)
+ }
+ if hosts["h.test"].box != "betty" {
+ t.Fatalf("this box is still excluded from its own routing; got %q", hosts["h.test"].box)
+ }
+}
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go
index 0bb51d361..3609e6813 100644
--- a/internal/host/edgesfile.go
+++ b/internal/host/edgesfile.go
@@ -81,6 +81,11 @@ type edgeHost struct {
// it remotely. Additive (omitempty).
Account string `json:"account,omitempty"`
App string `json:"app,omitempty"`
+ // Stamp is the claim's holder.stamp for the installation this box is
+ // serving (region HA spec 1 HA4). 0 = unstamped: no claim, or a writer
+ // that predates it. Two files claiming one host are settled by the
+ // higher stamp on the edge (homemap.go foldHomes).
+ Stamp int64 `json:"stamp,omitempty"`
}
// hostOwner folds the owners of every route served under one hostname. Any
diff --git a/internal/host/edgesfile_test.go b/internal/host/edgesfile_test.go
index 50051d133..d12c23686 100644
--- a/internal/host/edgesfile_test.go
+++ b/internal/host/edgesfile_test.go
@@ -1478,3 +1478,14 @@ func TestEdgesFileSeededGateIgnoresStoredHeadroom(t *testing.T) {
t.Fatalf("puts = %d, want 1", st.puts)
}
}
+
+func TestEdgesFileStampIsOmittedWhenZero(t *testing.T) {
+ b, _ := json.Marshal(edgeHost{Role: "home"})
+ if bytes.Contains(b, []byte("stamp")) {
+ t.Fatalf("zero stamp on the wire: %s", b)
+ }
+ b, _ = json.Marshal(edgeHost{Role: "home", Stamp: 41})
+ if !bytes.Contains(b, []byte(`"stamp":41`)) {
+ t.Fatalf("stamp missing: %s", b)
+ }
+}