amadan

carlos / platform Public

Merge branch 'region-hosting-armed' into main

claude pushed by paul@keymail.dev 66587fa8ae3c467653806e778e748fac074c7b0e parent 2e757faparent 78d5664
6 files changed, +283 −18
  • infra/modules/region-host/variables.tf +15 −1
  • internal/console/apiregions.go +3 −1
  • internal/console/instancesplacement.go +67 −16
  • internal/console/instancesplacement_test.go +75 −0
  • internal/edge/edge.go +28 −0
  • internal/edge/noactivator_test.go +95 −0

CI passed — run details

diff --git a/infra/modules/region-host/variables.tf b/infra/modules/region-host/variables.tf
index eefd8b9b5..1c9e3164e 100644
--- a/infra/modules/region-host/variables.tf
+++ b/infra/modules/region-host/variables.tf
@@ -34,7 +34,21 @@ variable "deployment_bucket_arn" {
}
variable "data_bucket_arn" {
- description = "ARN of THIS region's data bucket — the only data bucket the host role may reach."
+ description = <<-EOT
+ ARN of THIS region's data bucket — the only data bucket the host role may
+ reach.
+
+ Granting it is half the job. The box must also be TOLD the bucket, as
+ CARLOS_HIBERNATE_BUCKET (plus CARLOS_HIBERNATE_REGION) in
+ /etc/carlos/host.env, or its activator never arms and the box is an edge
+ that cannot host: it will accept an instance, mint the route, stage the
+ binary — and then serve 502 forever, because only an armed activator ever
+ creates the socket the edge is dialling. Five of tito's six regions sat in
+ exactly that state (discussion #110). The console now reads the box's
+ published data_bucket as its hosting signal, so a region that has not been
+ told is reported unarmed and refused at create rather than accepted and
+ stranded.
+ EOT
type = string
}
diff --git a/internal/console/apiregions.go b/internal/console/apiregions.go
index 59365e080..4c106bdd5 100644
--- a/internal/console/apiregions.go
+++ b/internal/console/apiregions.go
@@ -17,7 +17,9 @@ import (
// Codes are CARLOS placement codes, never AWS region strings. armed is
// EXACTLY the create door's predicate (#329's armedPlacements — a picker
// built on this can never offer what create would refuse): a box publishes
-// the code, and has a dial address. The home region appears under its NAME
+// the code, has a dial address, and can host (#110 — see armedPlacements
+// for why reachability alone advertised five regions that served 502).
+// The home region appears under its NAME
// (Config.HomeRegion) with armed true — create accepts it by normalizing
// the name to the record's "" — and is also the default. A deployment with
// no named home and no placement-publishing boxes answers empty lists, and
diff --git a/internal/console/instancesplacement.go b/internal/console/instancesplacement.go
index 41969cf15..18d9d9824 100644
--- a/internal/console/instancesplacement.go
+++ b/internal/console/instancesplacement.go
@@ -33,12 +33,17 @@ type servedBy struct {
// capacity, and "where is this instance" is the question, not "who is its
// write home" (that is the home map's question, internal/edge).
type edgesFilePlacementView struct {
- V int `json:"v"`
- Box string `json:"box"`
- EIP string `json:"eip"`
- Region string `json:"region"`
- Placement string `json:"placement"`
- Hosts map[string]json.RawMessage `json:"hosts"`
+ V int `json:"v"`
+ Box string `json:"box"`
+ EIP string `json:"eip"`
+ Region string `json:"region"`
+ Placement string `json:"placement"`
+ // DataBucket is the box's CARLOS_HIBERNATE_BUCKET, verbatim
+ // (internal/host/edgesfile.go writes it from that env and omits the key
+ // entirely when it is unset). It is read here as the box's own answer to
+ // "can I run an instance" — see armedPlacements.
+ DataBucket string `json:"data_bucket"`
+ Hosts map[string]json.RawMessage `json:"hosts"`
}
// loadEdgesJoin folds every readable edges file into host → the boxes that
@@ -98,22 +103,68 @@ func readEdgesFiles(ctx context.Context, s release.Store) []edgesFilePlacementVi
// armedPlacements folds the fleet's edges files into every published
// placement code and whether it can take an instance today (issue #329):
-// armed means some box publishing the code also has a dial address (an
-// EIP — a box nothing can reach is not a landing target), so set[code]
-// false is a region that EXISTS but cannot land one, which the regions
-// endpoint (#346) shows and the create door refuses alike. The second
-// return reports whether ANY file carried a placement code at all: false
-// means the fleet predates the field (or the read failed) and the create
-// gate stays DORMANT — the door then accepts what it always accepted, so
-// rolling boxes and console in either order is safe.
+// armed means some box publishing the code can be reached AND can host, so
+// set[code] false is a region that EXISTS but cannot land one, which the
+// regions endpoint (#346) shows and the create door refuses alike. The
+// second return reports whether ANY file carried a placement code at all:
+// false means the fleet predates the field (or the read failed) and the
+// create gate stays DORMANT — the door then accepts what it always
+// accepted, so rolling boxes and console in either order is safe.
+//
+// Reachable is an EIP: a box nothing can dial is not a landing target.
+//
+// Can host is a published data bucket, and it is the half discussion #110
+// paid for. Before it, armed was reachability alone — an EDGE-MEMBERSHIP
+// fact — while every consumer of this set reads it as "a region that can
+// take an instance right now". On tito those two diverged for five of six
+// regions: the boxes served TLS, converged routes and staged binaries
+// perfectly, so they were reachable and published their placement codes,
+// but none had been told a CARLOS_HIBERNATE_BUCKET. Their activators never
+// armed. An instance created into one of them was accepted, placed, staged
+// and routed, and then answered 502 forever, because the edge proxies a
+// hibernate-flagged route straight to a socket that only an armed
+// activator would ever create.
+//
+// data_bucket is the box's own answer and not an inference: internal/host's
+// edges-file pass writes CARLOS_HIBERNATE_BUCKET into it verbatim and omits
+// the key when that env is unset, and it is the SAME env, read in the same
+// process, that decides whether activator.ConfigFromEnv arms the activator
+// at all (internal/host/agent.go). So a published data bucket means that
+// box's activator is armed, which is exactly the question a region picker
+// is asking. It heals with no console change: the moment a region box's
+// host.env grows the variable, its next edges-file pass publishes the key
+// and the region arms itself.
+//
+// DORMANT until some box publishes one, the same roll-order rule the
+// placement field itself got and for the same reason. A fleet whose boxes
+// all predate the data_bucket field would otherwise go from "every region
+// armed" to "no region armed" the moment this console rolled, which is a
+// worse failure than the one it fixes — it would refuse creates into
+// regions that host perfectly well. With the gate dormant such a fleet
+// keeps the reachability-only answer it has always had.
func armedPlacements(ctx context.Context, s release.Store) (set map[string]bool, published bool) {
+ files := readEdgesFiles(ctx, s)
+ // The dormancy read, taken over every file before any verdict: one box
+ // publishing a data bucket is what makes the fleet's silence on the
+ // field meaningful for every OTHER box.
+ hostingKnown := false
+ for _, f := range files {
+ if f.DataBucket != "" {
+ hostingKnown = true
+ break
+ }
+ }
set = map[string]bool{}
- for _, f := range readEdgesFiles(ctx, s) {
+ for _, f := range files {
if f.Placement == "" {
continue
}
published = true
- set[f.Placement] = set[f.Placement] || f.EIP != ""
+ armed := f.EIP != ""
+ if hostingKnown {
+ armed = armed && f.DataBucket != ""
+ }
+ set[f.Placement] = set[f.Placement] || armed
}
return set, published
}
diff --git a/internal/console/instancesplacement_test.go b/internal/console/instancesplacement_test.go
new file mode 100644
index 000000000..656914dd1
--- /dev/null
+++ b/internal/console/instancesplacement_test.go
@@ -0,0 +1,75 @@
+package console
+
+import (
+ "context"
+ "testing"
+)
+
+// The hosting half of armed (discussion #110). Before it, armed was
+// reachability alone — an edge-membership fact — while every consumer reads
+// the set as "regions that can take an instance right now". On tito those two
+// diverged for five of six regions: reachable boxes that had never been told
+// a CARLOS_HIBERNATE_BUCKET, so their activators never armed, so an instance
+// placed there was accepted, staged and routed and then served 502 forever. A
+// box's published data_bucket is its own answer to "can I host", and it is
+// the same env that arms the activator.
+func TestArmedPlacementsRequiresHosting(t *testing.T) {
+ fx, _ := instanceFixture(t, "code-inst-armedhosting")
+ ctx := context.Background()
+ put := func(key, body string) {
+ t.Helper()
+ if _, err := fx.rel.Put(ctx, key, []byte(body), ""); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // Dormant: no box in the fleet publishes a data bucket at all, so the
+ // hosting gate must not fire and armed stays exactly the reachability
+ // answer these boxes have always had. A fleet whose boxes predate the
+ // field must not lose every region the day this console rolls.
+ put("control/edges/uk-box.json", `{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","hosts":{}}`)
+ put("control/edges/se-box.json", `{"v":1,"box":"se-box","eip":"","placement":"se","hosts":{}}`)
+ set, published := armedPlacements(ctx, fx.rel)
+ if !published {
+ t.Fatal("published = false, want true: both files carry a placement code")
+ }
+ if !set["uk"] {
+ t.Error("uk unarmed while the fleet publishes no data bucket at all — the gate must stay dormant")
+ }
+ if set["se"] {
+ t.Error("se armed with no EIP — the reachability half must still apply")
+ }
+
+ // One box publishes a data bucket. That is what makes every other box's
+ // silence on the field meaningful, so uk — reachable, but never told a
+ // hibernate bucket — is now correctly unarmed.
+ put("control/edges/ie-box.json", `{"v":1,"box":"ie-box","eip":"5.6.7.8","placement":"ie","data_bucket":"carlos-tito-ie-1","hosts":{}}`)
+ set, published = armedPlacements(ctx, fx.rel)
+ if !published {
+ t.Fatal("published = false, want true")
+ }
+ if !set["ie"] {
+ t.Error("ie unarmed: it is reachable and publishes a data bucket")
+ }
+ if set["uk"] {
+ t.Error("uk armed: reachable, but it publishes no data bucket, so its activator is not armed and an instance placed there would serve 502 forever")
+ }
+ if set["se"] {
+ t.Error("se armed: neither reachable nor hosting")
+ }
+
+ // A region arms itself the moment its box's host.env grows the variable —
+ // no console change and no console restart, just the box's next
+ // edges-file pass republishing the file it always publishes.
+ _, etag, err := fx.rel.Get(ctx, "control/edges/uk-box.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := fx.rel.Put(ctx, "control/edges/uk-box.json",
+ []byte(`{"v":1,"box":"uk-box","eip":"1.2.3.4","placement":"uk","data_bucket":"carlos-tito-uk-1","hosts":{}}`), etag); err != nil {
+ t.Fatal(err)
+ }
+ if set, _ = armedPlacements(ctx, fx.rel); !set["uk"] {
+ t.Error("uk still unarmed after publishing a data bucket")
+ }
+}
diff --git a/internal/edge/edge.go b/internal/edge/edge.go
index 4d86d5886..a58f17548 100644
--- a/internal/edge/edge.go
+++ b/internal/edge/edge.go
@@ -592,6 +592,34 @@ func (e *Edge) Reload() error {
if hib {
managed[t.Host] = true
}
+ // A hibernate-flagged instance route on a box with NO activator is
+ // unservable, and silently so — which is what discussion #110 cost.
+ // The row is real, the binary is staged, the socket path is derived,
+ // and nothing will ever create that socket, because only an armed
+ // activator spawns the child that binds it. Every request falls into
+ // unixProxy's dial and comes back as the stdlib ReverseProxy's bare
+ // 502 — a status that reads as "your backend is broken" for a
+ // backend that was never going to exist. On tito this ran for
+ // fifteen minutes across five regions before anyone opened an SSM
+ // session, and the box knew the answer the whole time.
+ //
+ // The fix for the CONSUMER half is the console's (armedPlacements
+ // now requires a published data bucket, so such a region is not
+ // offered and create refuses it). This is the box's half: say it
+ // out loud, name the variable, and let anyone reading the journal
+ // stop looking at the app.
+ //
+ // Throttled, because it is a standing condition — true on every
+ // reload for as long as the row and the missing env both exist — and
+ // keyed per host, a value the box itself controls (logThrottled's
+ // rule). Not an error and not a refusal to serve: `carlos edge` run
+ // without the agent has no activator by design, and taking the route
+ // out of the table would only trade a 502 for a 404.
+ if e.act == nil && t.Kind == "instance" && t.Hibernate && !viaTunnel {
+ e.logThrottled("no-activator "+t.Host, fmt.Sprintf(
+ "route %s is a hibernating instance but this box has no activator — nothing will ever create %s, so every request 502s. Set CARLOS_HIBERNATE_BUCKET in /etc/carlos/host.env and restart carlos-edge.",
+ t.Host, t.Socket))
+ }
m[t.Host] = upstream{proxy: proxy, route: t, hibernating: hib}
}
// Path routes, folded BEFORE aliases so an alias of a prefix-routed
diff --git a/internal/edge/noactivator_test.go b/internal/edge/noactivator_test.go
new file mode 100644
index 000000000..aba0af722
--- /dev/null
+++ b/internal/edge/noactivator_test.go
@@ -0,0 +1,95 @@
+package edge
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// A hibernate-flagged instance route on a box with no activator is
+// unservable, and before discussion #110 it was unservable in silence: the
+// row is real, the socket path is derived, and nothing will ever create that
+// socket, so every request comes back as unixProxy's bare stdlib 502. On tito
+// that ran for fifteen minutes across five regions before anyone opened an
+// SSM session, and the box knew the answer the whole time. Reload must name
+// the condition, the host, the socket nobody will bind, and the variable that
+// fixes it.
+func TestReloadNamesAHibernatingRouteWithNoActivator(t *testing.T) {
+ buf := captureLog(t)
+
+ reg := openReg(t)
+ // Socket-backed and hibernate-flagged: exactly what provision.Reconcile
+ // mints for a plain instance record, which is the shape that stranded.
+ if err := reg.Upsert(registry.Route{
+ Host: "region-probe.example.com", Kind: "instance",
+ Socket: "/run/carlos/region-probe.example.com.sock",
+ Status: "ready", Hibernate: true,
+ }); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ // A nil activator is the whole point: this is a box whose host.env never
+ // named a CARLOS_HIBERNATE_BUCKET, so internal/host armed nothing.
+ if _, err := New(reg, nil); err != nil {
+ t.Fatalf("New: %v", err)
+ }
+
+ got := buf.String()
+ for _, want := range []string{
+ "region-probe.example.com",
+ "/run/carlos/region-probe.example.com.sock",
+ "CARLOS_HIBERNATE_BUCKET",
+ "502",
+ } {
+ if !strings.Contains(got, want) {
+ t.Errorf("reload log does not mention %q — an operator reading the journal still cannot tell this from a broken app.\ngot: %s", want, got)
+ }
+ }
+}
+
+// The complaint is about the ACTIVATOR, not about instances in general: a box
+// with an activator armed is working exactly as designed and must say
+// nothing, or the line becomes noise on every healthy box in the fleet.
+func TestReloadIsSilentWhenTheActivatorIsArmed(t *testing.T) {
+ buf := captureLog(t)
+
+ reg := openReg(t)
+ if err := reg.Upsert(registry.Route{
+ Host: "healthy.example.com", Kind: "instance",
+ Socket: "/run/carlos/healthy.example.com.sock",
+ Status: "ready", Hibernate: true,
+ }); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ if _, err := New(reg, &fakeActivator{}); err != nil {
+ t.Fatalf("New: %v", err)
+ }
+
+ if got := buf.String(); strings.Contains(got, "no activator") {
+ t.Errorf("complained about an armed box: %s", got)
+ }
+}
+
+// An UNFLAGGED instance is proxied straight by design — that is the
+// deploy-safety property that lets the hibernate env ship fleet-wide as a
+// no-op before any host is flagged — so a box with no activator must not
+// complain about one either.
+func TestReloadIsSilentForAnUnflaggedInstance(t *testing.T) {
+ buf := captureLog(t)
+
+ reg := openReg(t)
+ if err := reg.Upsert(registry.Route{
+ Host: "unflagged.example.com", Kind: "instance",
+ Socket: "/run/carlos/unflagged.example.com.sock",
+ Status: "ready",
+ }); err != nil {
+ t.Fatalf("upsert: %v", err)
+ }
+ if _, err := New(reg, nil); err != nil {
+ t.Fatalf("New: %v", err)
+ }
+
+ if got := buf.String(); strings.Contains(got, "no activator") {
+ t.Errorf("complained about an unflagged instance: %s", got)
+ }
+}