carlos / platform Public

zdd-5 task 15: stage-relay retransmission

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude agent, operated by paul@keymail.dev 3e2280a34bdf4921bd1d31f078d60a27ec161f00 parent 04c1faa
3 files changed, +292 −2
  • internal/host/agent.go +16 −2
  • internal/host/stagerelay.go +110 −0
  • internal/host/stagerelay_test.go +166 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 218469d33..931569fa4 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -435,7 +435,12 @@ func runAgent(args []string, buildClaims ClaimFactory) {
if logSink != nil { // a typed nil in the interface would not be nil
rolloutSink = logSink
}
- armRollout(agentCtx, os.Getenv, svc, reg, dd, rolloutSink, openDeployStore)
+ if armRollout(agentCtx, os.Getenv, svc, reg, dd, rolloutSink, openDeployStore) != nil {
+ // Spec §7.7: a stage touch lost to a running root pass is asked
+ // again. Only with the rollout armed, so with CARLOS_SWAP off
+ // the stage path is byte-for-byte what it was.
+ armStageRelay(agentCtx, reg, dd)
+ }
// The residence-move machinery (issue #331), armed with the
// activator: the staging-fallback restore needs a same-region
// mirror client, and the move worker needs the service's quiesce
@@ -1430,13 +1435,22 @@ const stageRequestEvery = 15 * time.Second
// cannot be written costs latency, never correctness, because
// carlos-adopt.timer remains the floor underneath it.
func stageRequester(dd string) func(host, ver string, err error) {
- return markerRequester(dd, func(host, ver string, err error) {
+ ask := markerRequester(dd, func(host, ver string, err error) {
if err != nil {
log.Printf("agent: cannot request root stage for %s %s: %v", host, ver, err)
return
}
log.Printf("agent: %s needs %s staged and this process cannot write the version store — asked for a root pass", host, ver)
})
+ return func(host, ver string, err error) {
+ ask(host, ver, err)
+ // The stage relay (stagerelay.go) learns the head here and not in
+ // markerRequester, which mintRequester shares: a proof-mint request
+ // is not a head that failed to stage.
+ if r := armedStageRelay.Load(); r != nil {
+ r.note(host, ver)
+ }
+ }
}
// mintRequester is the mint asker's (proofask.go) request: the same marker
diff --git a/internal/host/stagerelay.go b/internal/host/stagerelay.go
new file mode 100644
index 000000000..98706337c
--- /dev/null
+++ b/internal/host/stagerelay.go
@@ -0,0 +1,110 @@
+package host
+
+import (
+ "context"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// stageRelay retransmits the stage request (spec §7.7). carlos-stage.path
+// does not act on a change made while carlos-stage.service is running, so a
+// touch that lands mid-pass is lost; this asks again every stageRequestEvery
+// until root has recorded an outcome for the head: its stamp shows the head
+// installed, or its refusal record names it. Armed only with the flag on.
+type stageRelay struct {
+ mu sync.Mutex
+ pending map[string]string // host -> the channel head a pass could not stage
+ get func(host string) (*registry.Route, error)
+ // refused reads root's refusal record for a host (release.ReadRefusal
+ // over the data dir): nil when there is none.
+ refused func(host string) (*release.Refusal, error)
+ touch func()
+}
+
+// note records the channel head a pass could not stage. A newer head for the
+// same host replaces the older one: only the newest is worth asking for.
+func (r *stageRelay) note(host, ver string) {
+ r.mu.Lock()
+ r.pending[host] = ver
+ r.mu.Unlock()
+}
+
+// tick touches the marker once if any head is still outstanding, and drops
+// the ones root has installed or refused, or whose route is gone. A row or a
+// record that cannot be read is no outcome, so it keeps the head pending.
+func (r *stageRelay) tick() bool {
+ r.mu.Lock()
+ snap := make(map[string]string, len(r.pending))
+ for h, v := range r.pending {
+ snap[h] = v
+ }
+ r.mu.Unlock()
+ outstanding := false
+ for host, ver := range snap {
+ rt, err := r.get(host)
+ if err != nil {
+ outstanding = true
+ continue
+ }
+ done := rt == nil || rt.Version == ver
+ if !done {
+ rec, rerr := r.refused(host)
+ if rerr != nil {
+ outstanding = true
+ continue
+ }
+ done = rec != nil && rec.Version == ver
+ }
+ if done {
+ r.mu.Lock()
+ if r.pending[host] == ver { // a newer head noted meanwhile stays
+ delete(r.pending, host)
+ }
+ r.mu.Unlock()
+ continue
+ }
+ outstanding = true
+ }
+ if outstanding {
+ r.touch()
+ }
+ return outstanding
+}
+
+// run ticks every interval until ctx ends. It is its own goroutine's body, so
+// a slow registry read delays the next retransmission and nothing else.
+func (r *stageRelay) run(ctx context.Context, every time.Duration) {
+ t := time.NewTicker(every)
+ defer t.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-t.C:
+ r.tick()
+ }
+ }
+}
+
+// armedStageRelay is the relay stageRequester feeds; nil while the rollout
+// is not armed, which leaves stageRequester exactly as it was.
+var armedStageRelay atomic.Pointer[stageRelay]
+
+// newStageRelay is the relay over root's real records: the registry's stamp
+// and the refusal files in the agent's data dir (its bake dir).
+func newStageRelay(reg *registry.Registry, dd string) *stageRelay {
+ return &stageRelay{pending: map[string]string{}, get: reg.Get,
+ refused: func(host string) (*release.Refusal, error) { return release.ReadRefusal(dd, host) },
+ touch: func() { touchStageMarker(dd) }}
+}
+
+// armStageRelay starts the retransmission loop; it ends with ctx.
+func armStageRelay(ctx context.Context, reg *registry.Registry, dd string) {
+ r := newStageRelay(reg, dd)
+ armedStageRelay.Store(r)
+ go r.run(ctx, stageRequestEvery)
+}
diff --git a/internal/host/stagerelay_test.go b/internal/host/stagerelay_test.go
new file mode 100644
index 000000000..884f083ff
--- /dev/null
+++ b/internal/host/stagerelay_test.go
@@ -0,0 +1,166 @@
+package host
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+func noRefusal(string) (*release.Refusal, error) { return nil, nil }
+
+func TestTheRelayAsksAgainUntilTheHeadIsInstalled(t *testing.T) {
+ rows := map[string]*registry.Route{"a": {Host: "a", Version: "v1"}, "b": {Host: "b", Version: "v1"}}
+ touches := 0
+ r := &stageRelay{pending: map[string]string{},
+ get: func(h string) (*registry.Route, error) { return rows[h], nil },
+ refused: noRefusal,
+ touch: func() { touches++ }}
+ if r.tick() || touches != 0 {
+ t.Fatal("touched with nothing outstanding")
+ }
+ r.note("a", "v2")
+ r.note("b", "v2")
+ if !r.tick() || touches != 1 {
+ t.Fatal("one touch per tick while heads are outstanding")
+ }
+ rows["a"].Version = "v2"
+ delete(rows, "b")
+ if r.tick() || touches != 1 {
+ t.Fatal("kept asking after root installed a's head and b's route went")
+ }
+ r.note("a", "v3")
+ r.get = func(string) (*registry.Route, error) { return nil, errors.New("database is locked") }
+ if !r.tick() {
+ t.Fatal("an unreadable row is not proof the head is installed")
+ }
+}
+
+// Root's refusal record is an outcome too: once root has refused a
+// (host, version), asking again only reruns the refusal.
+func TestTheRelayStopsAtRootsRefusal(t *testing.T) {
+ rows := map[string]*registry.Route{"a": {Host: "a", Version: "v1"}, "b": {Host: "b", Version: "v1"}}
+ refusals := map[string]*release.Refusal{"a": {Host: "a", Version: "v2", Kind: release.RefusalWrongProgram}}
+ touches := 0
+ r := &stageRelay{pending: map[string]string{},
+ get: func(h string) (*registry.Route, error) { return rows[h], nil },
+ refused: func(h string) (*release.Refusal, error) { return refusals[h], nil },
+ touch: func() { touches++ }}
+ r.note("a", "v2")
+ if r.tick() || touches != 0 {
+ t.Fatal("touched for a head root has refused")
+ }
+ r.note("b", "v3")
+ refusals["b"] = &release.Refusal{Host: "b", Version: "v2"} // an older refusal names another version
+ if !r.tick() || touches != 1 {
+ t.Fatal("a refusal of another version is no outcome for this head")
+ }
+ r.refused = func(string) (*release.Refusal, error) { return nil, errors.New("permission denied") }
+ if !r.tick() {
+ t.Fatal("an unreadable refusal record is not an outcome")
+ }
+}
+
+// A newer head replaces the older one for the same host, so the relay asks
+// for the newest only and stops when that one is installed.
+func TestTheRelayFollowsTheNewestHead(t *testing.T) {
+ rows := map[string]*registry.Route{"a": {Host: "a", Version: "v1"}}
+ touches := 0
+ r := &stageRelay{pending: map[string]string{},
+ get: func(h string) (*registry.Route, error) { return rows[h], nil },
+ refused: noRefusal,
+ touch: func() { touches++ }}
+ r.note("a", "v2")
+ r.note("a", "v3")
+ rows["a"].Version = "v2" // root installed the superseded head
+ if !r.tick() || touches != 1 {
+ t.Fatal("the superseded head's install must not end the ask for the newer one")
+ }
+ rows["a"].Version = "v3"
+ if r.tick() || touches != 1 {
+ t.Fatal("kept asking after the newest head was installed")
+ }
+}
+
+// The loop runs on its own goroutine, touches on its own interval and stops
+// when the agent's context is cancelled.
+func TestTheRelayLoopTicksAndStopsWithTheContext(t *testing.T) {
+ var touches atomic.Int32
+ r := &stageRelay{pending: map[string]string{},
+ get: func(h string) (*registry.Route, error) { return &registry.Route{Host: h, Version: "v1"}, nil },
+ refused: noRefusal,
+ touch: func() { touches.Add(1) }}
+ r.note("a", "v2")
+ ctx, cancel := context.WithCancel(context.Background())
+ done := make(chan struct{})
+ go func() { r.run(ctx, 5*time.Millisecond); close(done) }()
+ deadline := time.Now().Add(5 * time.Second)
+ for touches.Load() < 2 && time.Now().Before(deadline) {
+ time.Sleep(time.Millisecond)
+ }
+ if touches.Load() < 2 {
+ t.Fatal("the loop did not retransmit")
+ }
+ cancel()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("the loop outlived its context")
+ }
+ n := touches.Load()
+ time.Sleep(30 * time.Millisecond)
+ if touches.Load() != n {
+ t.Fatal("touched after the context was cancelled")
+ }
+}
+
+// stageRequester feeds the armed relay; mintRequester shares its marker and
+// throttle but a proof-mint request is not a stage-blocked head, so it must
+// not. With no relay armed (the flag off) stageRequester is as before.
+func TestOnlyStageRequestsFeedTheRelay(t *testing.T) {
+ dd := t.TempDir()
+ stageRequester(dd)("a", "v2", nil) // unarmed: no panic, marker as before
+ if _, err := os.Stat(filepath.Join(dd, stageRequestMarker)); err != nil {
+ t.Fatalf("marker not written with no relay armed: %v", err)
+ }
+ r := &stageRelay{pending: map[string]string{}, get: func(string) (*registry.Route, error) { return nil, nil }, refused: noRefusal, touch: func() {}}
+ armedStageRelay.Store(r)
+ t.Cleanup(func() { armedStageRelay.Store(nil) })
+ mintRequester(dd)("m", "v9", nil)
+ if len(r.pending) != 0 {
+ t.Fatalf("a mint request reached the relay: %v", r.pending)
+ }
+ stageRequester(dd)("a", "v2", nil)
+ if r.pending["a"] != "v2" {
+ t.Fatalf("a stage-blocked head did not reach the relay: %v", r.pending)
+ }
+}
+
+// armStageRelay reads root's real records: the registry row and the refusal
+// file in the data dir.
+func TestArmStageRelayReadsRootsRecords(t *testing.T) {
+ reg, dd := testRegistry(t), t.TempDir()
+ if err := reg.Upsert(registry.Route{Host: "a.example.com", Version: "v1"}); err != nil {
+ t.Fatalf("seed a route: %v", err)
+ }
+ r := newStageRelay(reg, dd)
+ r.note("a.example.com", "v2")
+ if !r.tick() {
+ t.Fatal("an uninstalled, unrefused head must be outstanding")
+ }
+ if _, err := os.Stat(filepath.Join(dd, stageRequestMarker)); err != nil {
+ t.Fatalf("tick did not touch the marker: %v", err)
+ }
+ if err := release.WriteRefusal(dd, release.Refusal{Schema: 1, Host: "a.example.com", Version: "v2", Kind: release.RefusalNotProven}); err != nil {
+ t.Fatal(err)
+ }
+ if r.tick() {
+ t.Fatal("kept asking after root wrote a refusal for the head")
+ }
+}