893b7d20f · carlos/platform

diff --git a/internal/host/agent.go b/internal/host/agent.go

index cab426503..d8a3a4d2e 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -2280,6 +2280,11 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu

// GET per tick. See restartSeed for why the baseline cannot wait for the

// first request to establish itself.

restartMem := newRestartMemory()

+ // `carlos instances retry` (zero-downtime spec §8.4) reaches the rollout

+ // manager through the narrowed-restart log; nil while it is not armed.

+ if m := armedRollout(); m != nil {

+ restartMem.retryHost = m.RetryHost

+ }

restartSeed(ctx, reg, store, dd, restartMem)

restartBounce := bounceFn(svc)

restartWake := wakeFn(svc)

diff --git a/internal/host/restart.go b/internal/host/restart.go

index 4341c93fe..e41a637cf 100644

--- a/internal/host/restart.go

+++ b/internal/host/restart.go

@@ -849,6 +849,16 @@ func restartRequestLeg(ctx context.Context, routes []registry.Route, store relea

// the first error the cause — they are the same cause in practice.

log.Printf("agent: restart check: %d apps unreadable: %v", len(readErrs), readErrs[0])

}

+ for _, host := range retryRequests(routes, scoped) {

+ if mem.retryHost == nil {

+ log.Printf("agent: retry requested for %s, dropped: rollout manager not armed on this box, so nothing is held to clear", host)

+ continue

+ }

+ if mem.retryHost(host) {

+ log.Printf("agent: retry requested for %s: its failure hold is cleared and its next attempt runs as a canary", host)

+ }

+ scoped = withoutRetries(scoped)

if len(moved) == 0 && len(scoped) == 0 {

return false

}

diff --git a/internal/host/restartscope.go b/internal/host/restartscope.go

index 55f1d3ba9..3c8309608 100644

--- a/internal/host/restartscope.go

+++ b/internal/host/restartscope.go

@@ -50,6 +50,11 @@ import (

type restartMemory struct {

gen map[[2]string]string // app -> restart-gen.json etag, or genAbsent

scope map[[2]string]string // app -> last applied request id, or release.RestartScopeNone

+

+ // retryHost clears one host's failure hold (rollout.Manager.RetryHost),

+ // for a RestartScopeRetry request. nil while the rollout manager is not

+ // armed: there is no hold to clear, and the request is dropped.

+ retryHost func(host string) bool

}

func newRestartMemory() *restartMemory {

@@ -139,6 +144,12 @@ func scopedHosts(routes []registry.Route, app [2]string, requests []release.Rest

seen := map[string]bool{}

var out []string

for _, req := range requests {

+ // A retry is not a restart: it is handled by retryRequests, and must

+ // neither cycle anything nor be logged as a selector this build does

+ // not understand.

+ if req.Scope.Kind == release.RestartScopeRetry {

+ continue

+ }

// Asked of the KIND alone, once, before any route is considered: an

// app with no routes on this box must not look like an app whose

// selector we failed to understand.

@@ -162,6 +173,48 @@ func scopedHosts(routes []registry.Route, app [2]string, requests []release.Rest

return out

}

+// retryRequests is the host-retry half of the narrowed log: the hosts of

+// this box's routes, in the requesting app, that outstanding retry requests

+// name. A host on another box names nothing here.

+func retryRequests(routes []registry.Route, work []scopedRestart) []string {

+ seen := map[string]bool{}

+ var out []string

+ for _, w := range work {

+ for _, req := range w.requests {

+ if req.Scope.Kind != release.RestartScopeRetry || seen[req.Scope.Value] {

+ continue

+ }

+ for _, rt := range routes {

+ if rt.Host == req.Scope.Value && rt.App == w.app[1] && release.RouteAccount(rt) == w.app[0] {

+ seen[rt.Host] = true

+ out = append(out, rt.Host)

+ break

+ }

+ sort.Strings(out)

+ return out

+}

+

+// withoutRetries drops retry requests, and apps left with none, so a serial

+// move that carried only retries asks root for no restart pass.

+func withoutRetries(work []scopedRestart) []scopedRestart {

+ var out []scopedRestart

+ for _, w := range work {

+ var keep []release.RestartRequest

+ for _, req := range w.requests {

+ if req.Scope.Kind != release.RestartScopeRetry {

+ keep = append(keep, req)

+ }

+ if len(keep) > 0 {

+ out = append(out, scopedRestart{app: w.app, requests: keep})

+ }

+ return out

+}

+

// execBounceable is the complement of what rootRestartRoute's systemctl arms

// can reach — no named unit, no unit backing — so the agent's half and the

// root half never cycle the same process twice. Split out of bounceExecRoutes

diff --git a/internal/host/restartscope_test.go b/internal/host/restartscope_test.go

index 7ce11f117..0829145e3 100644

--- a/internal/host/restartscope_test.go

+++ b/internal/host/restartscope_test.go

@@ -6,12 +6,15 @@ package host

// the narrowing, and that a selector this build cannot read cycles NOTHING.

import (

+ "bytes"

"context"

"encoding/json"

"errors"

+ "log"

"os"

"path/filepath"

"slices"

+ "strings"

"testing"

"github.com/carlosframework/platform/internal/registry"

@@ -575,3 +578,56 @@ func saveMarks(t *testing.T, path string, m *restartMarks) {

t.Fatalf("write marks: %v", err)

}

+

+// A retry request clears one host's hold through the rollout manager, cycles

+// nothing, and asks root for no restart pass.

+func TestRestartLegHandsARetryToTheRolloutManager(t *testing.T) {

+ ctx, s := genKit(t)

+ dd := t.TempDir()

+ routes := []registry.Route{execRoute("a.example.com", "bab", "eleven"), execRoute("b.example.com", "bab", "eleven")}

+ mem := newRestartMemory()

+ var retried []string

+ mem.retryHost = func(h string) bool { retried = append(retried, h); return true }

+ bounce := func(h string) bool { t.Errorf("%s was bounced for a retry", h); return true }

+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked {

+ t.Fatal("the baselining leg asked for a root pass")

+ }

+ askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeRetry, "b.example.com")

+ askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeRetry, "elsewhere.example.com")

+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked {

+ t.Fatal("a retry-only move asked for a root restart pass")

+ }

+ if !slices.Equal(retried, []string{"b.example.com"}) {

+ t.Fatalf("retried %v, want only this box's b.example.com", retried)

+ }

+}

+

+// With no manager armed (flag off, or a box that declined or refused it)

+// nothing is held: the request is consumed, logged, and cycles nothing.

+func TestRestartLegDropsARetryWithNoManager(t *testing.T) {

+ ctx, s := genKit(t)

+ dd := t.TempDir()

+ routes := []registry.Route{execRoute("a.example.com", "bab", "eleven")}

+ mem := newRestartMemory()

+ bounce := func(h string) bool { t.Errorf("%s was bounced", h); return true }

+ restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem)

+ askScoped(t, ctx, s, "bab", "eleven", release.RestartScopeRetry, "a.example.com")

+

+ var logbuf bytes.Buffer

+ log.SetOutput(&logbuf)

+ defer log.SetOutput(os.Stderr)

+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked {

+ t.Fatal("a retry with no manager asked for a root pass")

+ }

+ if got := logbuf.String(); !strings.Contains(got, "retry requested for a.example.com") || !strings.Contains(got, "rollout manager not armed") {

+ t.Fatalf("log = %q, want the dropped retry named", got)

+ }

+ // Consumed: the next pass does not see it again.

+ logbuf.Reset()

+ if asked := restartRequestLeg(ctx, routes, s, dd, bounce, nil, mem); asked {

+ t.Fatal("the second pass asked for a root pass")

+ }

+ if got := logbuf.String(); strings.Contains(got, "retry requested") {

+ t.Fatalf("the retry was seen again on the next pass: %q", got)

+ }

+}

diff --git a/internal/release/restartscope.go b/internal/release/restartscope.go

index deb1da287..f4de0dc4d 100644

--- a/internal/release/restartscope.go

+++ b/internal/release/restartscope.go

@@ -71,6 +71,13 @@ const (

RestartScopeHost = "host"

RestartScopeEnvironment = "environment"

RestartScopeApp = "app"

+

+ // RestartScopeRetry is `carlos instances retry <host>` (zero-downtime spec

+ // §8.4): Value is the host whose failure hold to clear. It is NOT a

+ // restart. Selects does not understand it, so a box built before it

+ // steps over it, and a box built after it hands it to the rollout manager

+ // instead of cycling anything (internal/host/restartscope.go).

+ RestartScopeRetry = "retry"

)

// restartScopeKeep bounds the log. Entries older than the last few dozen are

@@ -153,7 +160,7 @@ func (sc RestartScope) Selects(rt registry.Route) (selected, understood bool) {

// answered a typo with what exists.

func ValidRestartScope(sc RestartScope) error {

switch sc.Kind {

- case RestartScopeHost, RestartScopeEnvironment:

+ case RestartScopeHost, RestartScopeEnvironment, RestartScopeRetry:

default:

return fmt.Errorf("restart scope: unknown kind %q", sc.Kind)

}

diff --git a/internal/release/restartscope_test.go b/internal/release/restartscope_test.go

index caf3d9a8b..f122255d5 100644

--- a/internal/release/restartscope_test.go

+++ b/internal/release/restartscope_test.go

@@ -299,3 +299,15 @@ func TestAppendRestartScopeRefusesABadName(t *testing.T) {

t.Fatalf("temp dir: %v", err)

}

+

+// A retry is a narrowing a box understands only if it was built to: today's

+// Selects must not read it as a restart of anything.

+func TestRetryScopeIsWritableAndSelectsNothing(t *testing.T) {

+ sc := RestartScope{Kind: RestartScopeRetry, Value: "a.example.com"}

+ if err := ValidRestartScope(sc); err != nil {

+ t.Fatalf("ValidRestartScope(retry) = %v", err)

+ }

+ if sel, understood := sc.Selects(registry.Route{Host: "a.example.com"}); sel || understood {

+ t.Fatalf("Selects = %v, %v; a retry must never select a route for a restart", sel, understood)

+ }

+}

carlos / platform Public

zdd-6 task 11a: the retry selector on the narrowed-restart log, handed to the rollout manager