carlos / platform Public

region HA 2a: dial a box's private address within its region

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Paul Campbell pushed by claude 1 week ago aee7b1e75d72874f85a553a572aeafaf5c8ce225 parent e19a292

4 files changed, +151 −12

internal/edge/homemap.go +62 −9
internal/edge/homemap_private_test.go +84 −0
internal/edge/homemap_test.go +3 −3
internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv +2 −0

diff --git a/internal/edge/homemap.go b/internal/edge/homemap.go

index dfc1c85ec..504e6447a 100644

--- a/internal/edge/homemap.go

+++ b/internal/edge/homemap.go

@@ -80,10 +80,12 @@ import (

"os"

"slices"

"sort"

+ "strconv"

"strings"

"sync"

"time"

+ "github.com/carlosframework/platform/internal/edgesfile"

"github.com/carlosframework/platform/internal/release"

)

@@ -132,6 +134,39 @@ type edgesFileView struct {

Box string `json:"box"`

EIP string `json:"eip"`

Hosts map[string]edgesHostView `json:"hosts"`

+ // PrivateIP, PrivateCIDR, Region and Placement choose the dial address

+ // (region HA spec 2 §6, dialAddress). All are absent from a writer that

+ // predates them, and the dial is then EIP, as before.

+ PrivateIP string `json:"private_ip"`

+ PrivateCIDR string `json:"private_cidr"`

+ Region string `json:"region"`

+ Placement string `json:"placement"`

+// dialAddress is where this edge dials f's box (region HA spec 2 §6). With

+// privateDial (this edge's own region HA switch) on, a box that publishes

+// the same AWS region and placement as this one is dialled at its private

+// address, so the dial stays inside the region's network, but only when

+// that address passes edgesfile.PrivateAddress against the network this

+// box's own file publishes (census ruling region-ha-2a). Every other box,

+// and every box with privateDial off, is dialled at its public address,

+// as before. self is this box's own file, which selfKnown says was found.

+// "" is nothing to dial.

+func dialAddress(f, self edgesFileView, selfKnown, privateDial bool) string {

+ if privateDial && selfKnown && f.Region != "" && f.Placement != "" &&

+ f.Region == self.Region && f.Placement == self.Placement {

+ if addr, _ := edgesfile.PrivateAddress(f.PrivateIP, []string{self.PrivateCIDR}); addr != "" {

+ return addr

+ }

+ return f.EIP

+// regionHAFromEnv is CARLOS_REGION_HA as strconv.ParseBool reads it, the

+// reading every region HA switch makes.

+func regionHAFromEnv(getenv func(string) string) bool {

+ on, err := strconv.ParseBool(strings.TrimSpace(getenv("CARLOS_REGION_HA")))

+ return err == nil && on

}

// edgesHostView is the per-host object. An object rather than a bare string is

@@ -161,6 +196,8 @@ type edgesHostView struct {

// homeEntry is one folded answer: where to dial, and which box that is.

type homeEntry struct {

+ // eip is the address to dial: dialAddress's answer, the home box's

+ // private address within its region and its EIP from anywhere else.

eip string

box string

// platform carries the publisher's class mark through the fold.

@@ -238,6 +275,10 @@ type homeMap struct {

keyMu sync.RWMutex

keyCurrent []byte

keyNext []byte

+ // privateDial is this edge's region HA switch: only with it on does

+ // the fold dial a same-region box's private address (dialAddress).

+ privateDial bool

}

func newHomeMap(s release.Store, self string) *homeMap {

@@ -253,7 +294,9 @@ func (e *Edge) newHomeMap() *homeMap {

if e.staticStore == nil {

return nil

}

- return newHomeMap(e.staticStore, e.box)

+ m := newHomeMap(e.staticStore, e.box)

+ m.privateDial = regionHAFromEnv(os.Getenv)

+ return m

}

// Refresh rebuilds the host fold and re-reads the mesh keys. The two halves

@@ -330,7 +373,7 @@ func (m *homeMap) refreshHosts(ctx context.Context) error {

if failed > 0 {

m.logPartialPoll(failed, tried, lastErr)

}

- hosts, tied := foldHomes(next, m.self)

+ hosts, tied := foldHomes(next, m.self, m.privateDial)

if len(tied) > 0 {

// One claim revision names one executor, so two boxes publishing

// one host at the same stamp is a fault upstream. The fold stays

@@ -532,9 +575,11 @@ func (m *homeMap) logPartialPoll(failed, tried int, err error) {

// target would have the edge dial itself. Excluded by the file's own box

// field rather than by its object name, because the field is what the

// publisher stamps and what self is derived to match.

-// - NO EIP. Task 1 publishes a box with no discoverable public IP as

-// eip:"" so the DR runbook still gets its hosts list. There is nothing to

-// dial, so it is not a target.

+// - NO DIAL ADDRESS. Task 1 publishes a box with no discoverable public IP

+// as eip:"" so the DR runbook still gets its hosts list. There is

+// nothing to dial, so it is not a target. The address is dialAddress's:

+// a box in this box's own region is dialled at its private address when

+// it publishes one (region HA spec 2 §6), every other at its EIP.

// - A ROLE THIS BUILD DOES NOT KNOW. Only "home" routes. The role field

// exists so read-local roles can be added later; a reader that treated

// every role as a home would send traffic to a box that had declared it

@@ -546,28 +591,36 @@ func (m *homeMap) logPartialPoll(failed, tried int, err error) {

// split-and-wrong: the whole fleet aiming at one box is a condition that heals

// on the next publish, where half the fleet aiming at each is a condition that

// looks fine from any single box.

-func foldHomes(files map[string]edgesFileView, self string) (map[string]homeEntry, []string) {

+func foldHomes(files map[string]edgesFileView, self string, privateDial bool) (map[string]homeEntry, []string) {

keys := make([]string, 0, len(files))

for k := range files {

keys = append(keys, k)

}

slices.Sort(keys)

+ var own edgesFileView

+ ownKnown := false

+ for _, k := range keys {

+ if f := files[k]; self != "" && f.Box == self {

+ own, ownKnown = f, true

+ }

hosts := make(map[string]homeEntry)

for _, k := range keys {

f := files[k]

if f.Box == "" || f.Box == self {

continue

}

+ addr := dialAddress(f, own, ownKnown, privateDial)

for host, h := range f.Hosts {

if host == "" {

continue

}

switch h.Role {

case edgeRoleHome:

- // A dial target needs an address. A file with no EIP (a box

+ // A dial target needs an address. A file with none (a box

// with no discoverable public IP) still carries its cert

// halves below; its dial halves are simply not dialable.

- if f.EIP == "" {

+ if addr == "" {

if !h.Wildcard || !strings.HasPrefix(host, "*.") {

continue

}

@@ -583,7 +636,7 @@ func foldHomes(files map[string]edgesFileView, self string) (map[string]homeEntr

default:

continue

}

- in := homeEntry{eip: f.EIP, box: f.Box, platform: h.Platform && !h.Alias && !h.Wildcard, alias: h.Alias, wildcard: h.Wildcard, dial: h.Role == edgeRoleHome,

+ in := homeEntry{eip: addr, box: f.Box, platform: h.Platform && !h.Alias && !h.Wildcard, alias: h.Alias, wildcard: h.Wildcard, dial: h.Role == edgeRoleHome,

account: h.Account, app: h.App, stamp: h.Stamp}

prev, dup := hosts[host]

if !dup {

diff --git a/internal/edge/homemap_private_test.go b/internal/edge/homemap_private_test.go

new file mode 100644

index 000000000..ec2c9c018

--- /dev/null

+++ b/internal/edge/homemap_private_test.go

@@ -0,0 +1,84 @@

+package edge

+import "testing"

+// privateFleet publishes this edge's own file (annie, in ie, network

+// 10.0.0.0/16) and one file per case.

+func privateFleet(t *testing.T, m *homeMap) {

+ t.Helper()

+ putJSON(t, m.s, "control/edges/annie.json", `{"v":1,"box":"annie","eip":"1.1.1.1","private_ip":"10.0.1.1","private_cidr":"10.0.0.0/16","region":"eu-west-1","placement":"ie","hosts":{}}`)

+ putJSON(t, m.s, "control/edges/betty.json", `{"v":1,"box":"betty","eip":"2.2.2.2","private_ip":"10.0.1.2","private_cidr":"10.0.0.0/16","region":"eu-west-1","placement":"ie","hosts":{"b.test":{"role":"home"}}}`)

+ putJSON(t, m.s, "control/edges/carla.json", `{"v":1,"box":"carla","eip":"3.3.3.3","private_ip":"10.1.1.3","private_cidr":"10.1.0.0/16","region":"eu-west-2","placement":"uk","hosts":{"c.test":{"role":"home"}}}`)

+ putJSON(t, m.s, "control/edges/dora.json", `{"v":1,"box":"dora","eip":"4.4.4.4","region":"eu-west-1","placement":"ie","hosts":{"d.test":{"role":"home"}}}`)

+ putJSON(t, m.s, "control/edges/edna.json", `{"v":1,"box":"edna","eip":"5.5.5.5","private_ip":"10.0.1.5","region":"eu-west-1","placement":"fleet/shop","hosts":{"e.test":{"role":"home"}}}`)

+ putJSON(t, m.s, "control/edges/fern.json", `{"v":1,"box":"fern","eip":"","private_ip":"10.0.1.6","region":"eu-west-1","placement":"ie","hosts":{"f.test":{"role":"home"}}}`)

+ // Addresses a reader refuses: each falls back to the public address.

+ putJSON(t, m.s, "control/edges/gail.json", `{"v":1,"box":"gail","eip":"7.7.7.7","private_ip":"10.0.1","region":"eu-west-1","placement":"ie","hosts":{"g.test":{"role":"home"}}}`)

+ putJSON(t, m.s, "control/edges/hana.json", `{"v":1,"box":"hana","eip":"8.8.8.8","private_ip":"169.254.169.254","private_cidr":"169.254.0.0/16","region":"eu-west-1","placement":"ie","hosts":{"h.test":{"role":"home"}}}`)

+ putJSON(t, m.s, "control/edges/iris.json", `{"v":1,"box":"iris","eip":"9.9.9.9","private_ip":"10.9.1.9","private_cidr":"10.9.0.0/16","region":"eu-west-1","placement":"ie","hosts":{"i.test":{"role":"home"}}}`)

+ refreshHomeMap(t, m)

+// Region HA spec 2 §6: with this edge's region HA on, a box in the same

+// region is dialled at its private address and a box in another region at

+// its own public address. An address that fails validation against this

+// edge's own network is refused, and the dial is public.

+func TestHomeMapDialsPrivateWithinTheRegionAndPublicBetween(t *testing.T) {

+ m := newHomeMap(meshTestStore(t), "annie")

+ m.privateDial = true

+ privateFleet(t, m)

+ for host, want := range map[string]string{

+ "b.test": "10.0.1.2", // same region and placement: private

+ "c.test": "3.3.3.3", // another region: its own public address

+ "d.test": "4.4.4.4", // a writer with no private_ip: public, as before

+ "e.test": "5.5.5.5", // same AWS region, another placement: public

+ "f.test": "10.0.1.6", // no public address, but private within the region

+ "g.test": "7.7.7.7", // a malformed private address: refused

+ "h.test": "8.8.8.8", // the metadata range: refused

+ "i.test": "9.9.9.9", // outside this region's network: refused

+ } {

+ if addr, _, ok := m.Home(host); !ok || addr != want {

+ t.Errorf("%s dials %q (ok=%v), want %q", host, addr, ok, want)

+ }

+ // An edge whose own file is not in the map knows no region or network

+ // of its own: every dial is public, and a box with no public address

+ // is no target.

+ ghost := newHomeMap(m.s, "ghost")

+ ghost.privateDial = true

+ refreshHomeMap(t, ghost)

+ if addr, _, _ := ghost.Home("b.test"); addr != "2.2.2.2" {

+ t.Errorf("without its own file: b.test dials %q, want 2.2.2.2", addr)

+ }

+ if _, _, ok := ghost.Home("f.test"); ok {

+ t.Error("without its own file, a box with no public address became a target")

+ }

+// With this edge's region HA off, a peer publishing private_ip changes

+// nothing: every dial is public, as on main, during a mixed roll or a

+// rollback.

+func TestHomeMapWithRegionHAOffDialsPublicOnly(t *testing.T) {

+ m := newHomeMap(meshTestStore(t), "annie")

+ privateFleet(t, m)

+ for host, want := range map[string]string{"b.test": "2.2.2.2", "c.test": "3.3.3.3", "d.test": "4.4.4.4"} {

+ if addr, _, ok := m.Home(host); !ok || addr != want {

+ t.Errorf("%s dials %q (ok=%v), want %q", host, addr, ok, want)

+ }

+ if _, _, ok := m.Home("f.test"); ok {

+ t.Error("a box with only a private address became a target with region HA off")

+ }

+func TestEdgeHomeMapFollowsRegionHA(t *testing.T) {

+ e := &Edge{staticStore: meshTestStore(t), box: "annie"}

+ t.Setenv("CARLOS_REGION_HA", "")

+ if e.newHomeMap().privateDial {

+ t.Fatal("region HA unset dials private")

+ }

+ t.Setenv("CARLOS_REGION_HA", "1")

+ if !e.newHomeMap().privateDial {

+ t.Fatal("region HA on does not dial private")

+ }

diff --git a/internal/edge/homemap_test.go b/internal/edge/homemap_test.go

index 5c9d5e0e9..cdf518660 100644

--- a/internal/edge/homemap_test.go

+++ b/internal/edge/homemap_test.go

@@ -1715,7 +1715,7 @@ func TestHomeMapReportsTwoBoxesAtTheSameStamp(t *testing.T) {

"control/edges/betty.json": file("betty", "2.2.2.2", map[string]edgesHostView{

"tied.test": {Role: "home", Stamp: 7}, "fine.test": {Role: "home", Stamp: 9}, "plain.test": {Role: "home"}}),

}

- hosts, tied := foldHomes(files, "carol")

+ hosts, tied := foldHomes(files, "carol", false)

if len(tied) != 1 || tied[0] != "tied.test" {

t.Fatalf("tied = %v, want [tied.test]: unstamped duplicates are normal (static sites run on every box) and are not reported", tied)

}

@@ -1735,7 +1735,7 @@ func TestHomeMapReportsATieBelowTheWinner(t *testing.T) {

"control/edges/betty.json": file("betty", "2.2.2.2", map[string]edgesHostView{"h.test": {Role: "home", Stamp: 7}}),

"control/edges/carol.json": file("carol", "3.3.3.3", map[string]edgesHostView{"h.test": {Role: "home", Stamp: 7}}),

}

- hosts, tied := foldHomes(files, "dave")

+ hosts, tied := foldHomes(files, "dave", false)

if len(tied) != 1 || tied[0] != "h.test" {

t.Fatalf("tied = %v, want [h.test]", tied)

}

@@ -1751,7 +1751,7 @@ func TestHomeMapReportsATieWithThisBoxItself(t *testing.T) {

"control/edges/annie.json": file("annie", "1.1.1.1", map[string]edgesHostView{"h.test": {Role: "home", Stamp: 7}}),

"control/edges/betty.json": file("betty", "2.2.2.2", map[string]edgesHostView{"h.test": {Role: "home", Stamp: 7}}),

}

- hosts, tied := foldHomes(files, "annie")

+ hosts, tied := foldHomes(files, "annie", false)

if len(tied) != 1 || tied[0] != "h.test" {

t.Fatalf("tied = %v, want [h.test]", tied)

}

diff --git a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv

index 792db1fc7..3bfa644f1 100644

--- a/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv

+++ b/internal/provision/testdata/plan14-root-reads-region-ha-2a.tsv

@@ -5,10 +5,12 @@

# Task 5: the edges file's headroom carries the box's total memory and data volume size (headroomFor and capacityMiB, internal/host/headroom.go; edgeHeadroom's two additive fields). headroomFor calls the existing readMemBytes and diskBytes and adds no read of its own. Envelope-only: readHeadroom and readMemBytes are unchanged, no root-read unit changed or was added, and the legacy seal stands.

# Task 6: hostops.ExecutorChooser applies spec 2's policy in Choose and Permits (creates and relocation targets) and names the placeable set in PlaceableBoxes and Capacity. ExecutorChooser.eligible, the accepted-legacy unit (merge ruling 2), is unchanged: the policy runs over its result. Envelope-only, and the legacy seal stands.

# Task 7: the box's own addresses (internal/host boxaddress.go, the provider-neutral source over CARLOS_PRIVATE_IP with CARLOS_PRIVATE_CIDR, and CARLOS_PUBLIC_IP, all from root-owned host.env; awsaddr.go, the IMDS adapter, which reads the VPC's CIDR blocks too) and the edges file's private_ip and private_cidr, validated before they are written by edgesfile.PrivateAddress (census ruling region-ha-2a, condition 1). edgesFileWriteClaims (authority, F,J1) is re-digested with its class, task, basis and sites unchanged: with region HA on it reads the address source before every publication and no longer holds a blank EIP. It adds no registry, /data, release or claim read, and nothing runs as root that did not before; the legacy seal stands.

+# Task 8: with the reader's own region HA on, forwarding dials a same-region box's private address when it passes edgesfile.PrivateAddress against the reader's own published network, and every other box at its public address (internal/edge homemap.go: dialAddress, regionHAFromEnv, edgesFileView's private_ip, private_cidr, region and placement). The home map is not a root-read unit. Envelope-only, and the legacy seal stands.

@coverage internal/lbhealth ad1211a01a4eb3d02cd8ed95cbff0f8b17b3976ce9c0279ebd2cae2f20c07124

@coverage internal/hostactors 6736698380a247764ada687f2476f89382fc32a1183a50c06afe75967d716eab

@coverage internal/boxchoice 1bfd561bcb48b51e60e63f99e086fd44d3b1be2160e28b3ed0253a14bbb9ff66

@coverage internal/edgesfile f62c3973bb48fa36f561d26f496c0ba9a14fe882fe5843867e9f7a9eaf0abbdf

@coverage internal/host 75e0a977c0f286966f2fb1ebbca10aa40f3795117958bf82e80cf545d6a776fa

@coverage internal/hostops 3dc23aaff7bdae0341a8c9a6fbac076a2f57ca1e4676d83543433ff9ad7d1c79

+@coverage internal/edge 2c474d685471295daf20f061e595becab78175c5754dbef0d2187f0f8519fbe3

internal/host/edgesfile.go edgesFileWriteClaims authority F,J1 root-route;kept=F;scope=protected-decision-only registry:Account@10835,registry:Alias@5297,registry:Alias@5332,registry:Alias@5754,registry:Alias@5911,registry:Alias@5981,registry:Alias@6013,registry:Alias@6482,registry:Alias@8772,registry:App@10847,registry:DNSStatus@5429,registry:DNSStatus@5920,registry:Host@10480,registry:Host@2878,registry:Host@5389,registry:Host@6234,registry:Host@6567,registry:Host@8819,registry:Host@8866,registry:Host@8916,registry:Host@8951,registry:Host@9000,registry:Platform@2921,registry:Provisioned@2935,registry:Replica@2959,registry:Status@877,whole:Route@3212,whole:Route@323,whole:Route@3505,whole:Route@351,whole:RouteAlias@5282,whole:RouteAlias@8725,whole:RoutePath@3520,whole:WildcardDomain@8905 8c7cdffba66886b92f9a2ac663a86dbdd5f80699afd2e6afb4571da916c727ad