amadan

carlos / platform Public

edgesfile: a fold that joins nothing says so, and the doc names why it can

The liveness join reads edges files and box snapshots out of the ONE store
it is handed, but the two halves are written through different ones: edges
via openDeployStore (CARLOS_DEPLOYMENT_BUCKET, unprefixed), snapshots via
openStatusStore (CARLOS_STATUS_BUCKET when set, rebased under
CARLOS_STATUS_PREFIX). Same store by default, not by guarantee —
README.md's fleet section suggests a separate status bucket outright.

On a deployment that splits them the join can never match, every edges file
is dropped as unproven, and the answer is the env list: safe (that is
exactly the pre-fold behaviour) but INVISIBLE — no error, no log, and no
test can catch it, because the cause is configuration.

So: one line when files were found and not one of them joined, naming the
two variables to look at. It doubles as the runtime tripwire for box-label
drift between the two writers, which the identity test can only pin at the
function level. Silent when there are no files at all (an ordinary state)
and silent on a partial miss (the staleness rule working).
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Paul Campbell pushed by paul@keymail.dev c15e6d355cba98ecbeee245e1d1e8faa76bf4afc parent ec3767c
2 files changed, +98 −0
  • internal/edgesfile/fold.go +30 −0
  • internal/edgesfile/fold_test.go +68 −0
diff --git a/internal/edgesfile/fold.go b/internal/edgesfile/fold.go
index 971f50289..dcbb33778 100644
--- a/internal/edgesfile/fold.go
+++ b/internal/edgesfile/fold.go
@@ -28,6 +28,21 @@ import (
// statusTick's WriteBox is unconditional every tick, so BoxStale is a real
// liveness signal. Without one, a retired box's lingering edges file would
// keep a DEAD customer domain reading "pointing" forever.
+//
+// LIMITATION, and the reason for the log line below: that join reads both
+// halves out of the ONE store it is handed, but the two halves are written
+// through different ones. Edges files go through openDeployStore
+// (CARLOS_DEPLOYMENT_BUCKET, unprefixed); box snapshots go through
+// openStatusStore, which honours CARLOS_STATUS_BUCKET and rebases everything
+// under CARLOS_STATUS_PREFIX. They are the same store by DEFAULT, not by
+// guarantee — README.md's fleet section suggests a separate status bucket
+// outright — and on a deployment that splits them nothing here can see the
+// other half, so every edges file is dropped as unproven and the answer is
+// the env list. That degradation is SAFE (env-only is exactly the
+// pre-fold behaviour) but it is INVISIBLE, which is the failure class this
+// whole rule exists to avoid. So when files were found and none of them
+// joined, say so — configuration is the likely cause and no test can catch
+// it.
func EdgeIPs(ctx context.Context, s release.Store, env []string, now time.Time) []string {
out := append([]string(nil), env...)
add := func(ip string) {
@@ -54,6 +69,7 @@ func EdgeIPs(ctx context.Context, s release.Store, env []string, now time.Time)
log.Printf("edgesfile: could not list %s (%v) — edge set falls back to the configured list", Prefix, err)
return out
}
+ joined := 0
for _, key := range keys {
body, _, err := s.Get(ctx, key)
if err != nil {
@@ -68,7 +84,21 @@ func EdgeIPs(ctx context.Context, s release.Store, env []string, now time.Time)
if !live[f.Box] {
continue // no fresh snapshot: not evidence of a live edge
}
+ joined++
add(f.EIP)
}
+ // Files exist and not one of them proved live. A single retired box is
+ // the ordinary reading of a partial miss, but a TOTAL miss usually is
+ // not: either every box in the deployment went dark at once, or the two
+ // halves of the join are looking at different buckets (see the doc
+ // comment), or the box names on the two sides have drifted apart. All
+ // three are silent otherwise — the answer is simply the env list, with
+ // no error to return and nothing red anywhere. This line is the
+ // tripwire. Nothing is logged when there are no files at all: a
+ // deployment that has not published any is an ordinary state, not a
+ // fault.
+ if len(keys) > 0 && joined == 0 {
+ log.Printf("edgesfile: %d edges file(s) under %s but no fresh box snapshot matched any of them — edge set is the configured list only (are edges and status in different buckets? see CARLOS_STATUS_BUCKET/CARLOS_STATUS_PREFIX)", len(keys), Prefix)
+ }
return out
}
diff --git a/internal/edgesfile/fold_test.go b/internal/edgesfile/fold_test.go
index e112960b8..4635a1d00 100644
--- a/internal/edgesfile/fold_test.go
+++ b/internal/edgesfile/fold_test.go
@@ -1,9 +1,12 @@
package edgesfile
import (
+ "bytes"
"context"
"errors"
+ "log"
"slices"
+ "strings"
"testing"
"time"
@@ -144,3 +147,68 @@ func (brokenStore) Put(context.Context, string, []byte, string) (string, error)
func (brokenStore) List(context.Context, string) ([]string, error) { return nil, errBrokenStore }
func (brokenStore) ListDirs(context.Context, string) ([]string, error) { return nil, errBrokenStore }
func (brokenStore) Delete(context.Context, string) error { return errBrokenStore }
+
+// Edges files exist and NONE of them proved live: the answer degrades to the
+// env list with no error and nothing red, so the log line is the only thing
+// an operator can see. The likeliest cause is configuration rather than a
+// dead fleet — edges files are written through openDeployStore while box
+// snapshots go through openStatusStore, which honours CARLOS_STATUS_BUCKET
+// and CARLOS_STATUS_PREFIX, so the two halves of this join are the same
+// store by default but not by guarantee. Operational, not contractual: what
+// is pinned is that the line appears at all and names where to look.
+func TestEdgeIPsSaysSoWhenNothingJoined(t *testing.T) {
+ buf := captureLog(t)
+ s := foldTestStore(t)
+ now := time.Date(2026, 8, 27, 12, 0, 0, 0, time.UTC)
+ putEdgesFile(t, s, "annie", `{"v":1,"box":"annie","eip":"1.1.1.1","hosts":{}}`)
+ putEdgesFile(t, s, "betty", `{"v":1,"box":"betty","eip":"2.2.2.2","hosts":{}}`)
+ // No snapshots at all — exactly what a split-bucket deployment looks
+ // like from in here.
+ if got := EdgeIPs(context.Background(), s, []string{"9.9.9.9"}, now); !slices.Equal(got, []string{"9.9.9.9"}) {
+ t.Fatalf("EdgeIPs = %v, want env only", got)
+ }
+ line := buf.String()
+ if !strings.Contains(line, "no fresh box snapshot matched") {
+ t.Errorf("log = %q, want the total-miss tripwire", line)
+ }
+ if !strings.Contains(line, "CARLOS_STATUS_BUCKET") {
+ t.Errorf("log = %q, want it to name where to look", line)
+ }
+}
+
+// A deployment that has published no edges files yet is an ordinary state,
+// not a fault — and so is a partial miss, where one retired box drops out and
+// the rest join. Neither may cry wolf, or the line stops being read.
+func TestEdgeIPsStaysQuietWhenThereIsNothingToJoinOrSomethingDid(t *testing.T) {
+ now := time.Date(2026, 8, 27, 12, 0, 0, 0, time.UTC)
+
+ buf := captureLog(t)
+ empty := foldTestStore(t)
+ EdgeIPs(context.Background(), empty, []string{"9.9.9.9"}, now)
+ if strings.Contains(buf.String(), "no fresh box snapshot matched") {
+ t.Errorf("log = %q, want silence: no edges files is not a fault", buf.String())
+ }
+
+ buf2 := captureLog(t)
+ s := foldTestStore(t)
+ putEdgesFile(t, s, "annie", `{"v":1,"box":"annie","eip":"1.1.1.1","hosts":{}}`)
+ putEdgesFile(t, s, "retired", `{"v":1,"box":"retired","eip":"3.3.3.3","hosts":{}}`)
+ putFreshBox(t, s, "annie", now)
+ putFreshBox(t, s, "retired", now.Add(-72*time.Hour))
+ EdgeIPs(context.Background(), s, nil, now)
+ if strings.Contains(buf2.String(), "no fresh box snapshot matched") {
+ t.Errorf("log = %q, want silence: one box joined, which is the rule working", buf2.String())
+ }
+}
+
+// captureLog redirects the standard logger for one test — the same
+// save/restore-under-Cleanup shape internal/steering's tests use.
+func captureLog(t *testing.T) *bytes.Buffer {
+ t.Helper()
+ var buf bytes.Buffer
+ prev, flags := log.Writer(), log.Flags()
+ log.SetOutput(&buf)
+ log.SetFlags(0)
+ t.Cleanup(func() { log.SetOutput(prev); log.SetFlags(flags) })
+ return &buf
+}