amadan

carlos / platform Public

zdd-5 task 14: refusing mode does not ack an awake managed host as restarted

An awake managed instance still runs its old config while managed starts are
refused, so its config restart now answers a fixed error and the ack carries
RestartError with Restarted false (spec 8.1). Sleepers and unmanaged hosts are
unchanged.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 04c1faa329cb7e9edee273c4a2375854ad7c7387 parent aa8295c
3 files changed, +121 −8
  • internal/host/agent.go +1 −1
  • internal/host/rollout.go +37 −4
  • internal/host/rollout_test.go +83 −3
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 4fddde110..218469d33 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -1128,7 +1128,7 @@ func syncExecSerialized(ctx context.Context, reg *registry.Registry, store relea
// CARLOS_SWAP=overlap: the scheduler restarts, paced, and acknowledges a
// revision once it is active; refusing: no restart. configSyncMu above
// is what makes ConfigDelivered serial per host.
- if cb, ok := rolloutConfigCallbacks(ctx, store); ok {
+ if cb, ok := rolloutConfigCallbacks(ctx, store, managedAwake(svc, reg)); ok {
return appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, dd, os.Getenv("PLATFORM_APPS_DOMAIN"), cb, opts...)
}
return appconfig.SyncExec(ctx, reg, store, priv, dd, bounce(svc), os.Getenv("PLATFORM_APPS_DOMAIN"), configAckFunc(ctx, store, "agent"), opts...)
diff --git a/internal/host/rollout.go b/internal/host/rollout.go
index 09eaa4017..81a0c3d0d 100644
--- a/internal/host/rollout.go
+++ b/internal/host/rollout.go
@@ -2,6 +2,7 @@ package host
import (
"context"
+ "errors"
"log"
"os"
"path/filepath"
@@ -370,9 +371,15 @@ func fastConvergeLeg(svc *activator.Service, reg *registry.Registry) func() {
// syncExecSerialized runs today's SyncExec unchanged. Armed: the scheduler
// restarts exec instances, paced, and a revision is acknowledged only once
// it is active (spec §8.1, §10); appconfig calls Delivered after the file is
-// written (syncExecHostClaimed). Refusing: delivered and acknowledged as
-// today, but no restart, because the wake after it would be refused.
-func rolloutConfigCallbacks(ctx context.Context, store release.Store) (appconfig.Callbacks, bool) {
+// written (syncExecHostClaimed). Refusing: delivered and acknowledged at
+// delivery, but no restart, because the wake after it would be refused. An
+// AWAKE managed host (awake says which) still runs its old config, so its
+// restart answers an error and the ack carries RestartError with Restarted
+// false rather than claiming a restart that did not happen (spec §8.1). A
+// sleeper, and any unmanaged host, keep the no-op restart and so the ack they
+// got before. (The exec pass never sets Hibernating; only the unit pass in
+// appconfig/sync.go does, from the route's Hibernate flag.) awake may be nil.
+func rolloutConfigCallbacks(ctx context.Context, store release.Store, awake func(host string) bool) (appconfig.Callbacks, bool) {
if m := armedRollout(); m != nil {
return appconfig.Callbacks{
Restart: func(string) error { return nil },
@@ -381,7 +388,13 @@ func rolloutConfigCallbacks(ctx context.Context, store release.Store) (appconfig
}, true
}
if rolloutUnavailable() != "" {
- return appconfig.Callbacks{Restart: func(string) error { return nil }, Ack: configAckFunc(ctx, store, "agent")}, true
+ restart := func(host string) error {
+ if awake != nil && awake(host) {
+ return errors.New(rolloutRestartRefused)
+ }
+ return nil
+ }
+ return appconfig.Callbacks{Restart: restart, Ack: configAckFunc(ctx, store, "agent")}, true
}
return appconfig.Callbacks{}, false
}
@@ -437,3 +450,23 @@ func rolloutStatusProbe() *status.RolloutReport {
r := m.Status()
return &r
}
+
+// rolloutRestartRefused is the RestartError an awake managed host's config
+// ack carries while managed starts are refused. It is fixed text: the cause
+// goes to the agent's log, never to the member (plan 6 words it).
+const rolloutRestartRefused = "restart refused: rolling restarts are unavailable on this box"
+
+// managedAwake reports whether host is a managed exec instance the activator
+// has a live entry for (waking, awake or stopping, as Owns counts them), so
+// one whose process may still hold its old config. A host that is not in the
+// registry, not managed, or not owned is not awake; nor is anything with no
+// activator.
+func managedAwake(svc *activator.Service, reg *registry.Registry) func(host string) bool {
+ return func(host string) bool {
+ if svc == nil || reg == nil || !svc.Owns(host) {
+ return false
+ }
+ rt, err := reg.Get(host)
+ return err == nil && rt != nil && rollout.Managed(*rt)
+ }
+}
diff --git a/internal/host/rollout_test.go b/internal/host/rollout_test.go
index d5ff0daef..c0cfab6de 100644
--- a/internal/host/rollout_test.go
+++ b/internal/host/rollout_test.go
@@ -76,7 +76,7 @@ func legsAreTodays(t *testing.T, svc *activator.Service, reg *registry.Registry)
if b := adoptBounceFn(svc); b == nil || funcPtr(b) != funcPtr(svc.BounceAdopted) {
t.Error("the adopt bounce is not BounceAdopted")
}
- if _, ok := rolloutConfigCallbacks(context.Background(), release.NewDirStore(t.TempDir())); ok {
+ if _, ok := rolloutConfigCallbacks(context.Background(), release.NewDirStore(t.TempDir()), nil); ok {
t.Error("config sync left today's SyncExec")
}
ctx, s := genKit(t)
@@ -176,7 +176,7 @@ func TestTheFlagOnArmsTheScheduler(t *testing.T) {
if fastConvergeLeg(svc, reg) == nil {
t.Fatal("no converge leg: the scheduler's pass must run")
}
- cb, ok := rolloutConfigCallbacks(context.Background(), store)
+ cb, ok := rolloutConfigCallbacks(context.Background(), store, nil)
if !ok || cb.Delivered == nil || cb.Ack == nil || cb.Restart == nil || cb.Restart("a.example.com") != nil {
t.Fatalf("config callbacks = %+v, %v: want the scheduler's delivery and deferred ack, and no restart", cb, ok)
}
@@ -264,7 +264,7 @@ func TestUnavailableStateRefusesManagedStarts(t *testing.T) {
t.Fatal("OnGen set with no scheduler to hear it")
}
store := release.NewDirStore(t.TempDir())
- cb, ok := rolloutConfigCallbacks(context.Background(), store)
+ cb, ok := rolloutConfigCallbacks(context.Background(), store, nil)
if !ok || cb.Delivered != nil || cb.Restart == nil || cb.Restart("a.example.com") != nil || cb.Ack == nil {
t.Fatalf("config callbacks = %+v, %v: want delivery and the ack as today, and no restart", cb, ok)
}
@@ -508,3 +508,83 @@ func TestThePolicyLoaderFailsOnAnyStoreError(t *testing.T) {
t.Fatalf("an undecodable config object read as %+v", m)
}
}
+
+// A refusing box must not tell a member an awake managed instance was
+// restarted: the instance still runs its old config and nothing will restart
+// it (spec §8.1, an ack only once active). Its ack carries a fixed
+// RestartError and Restarted stays false. A sleeper reads the file on its
+// next wake, which is refused or not on its own terms, so its ack is as it
+// is today: no error.
+func TestRefusingConfigSyncDoesNotAckAnAwakeManagedHostAsRestarted(t *testing.T) {
+ ctx := context.Background()
+ svc, reg, dd := rolloutFixture(t, true)
+ _, pub, err := release.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ noStore := func(context.Context) (release.Store, error) {
+ return nil, errors.New("no deployment store: secret detail")
+ }
+ if m := armRollout(ctx, overlapEnv(pub), svc, reg, dd, nil, noStore); m != nil {
+ t.Fatal("armed without a store")
+ }
+ store := release.NewDirStore(t.TempDir())
+ privText, _, err := appconfig.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ priv, err := appconfig.ParsePrivateKey(privText, "test")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := appconfig.SetEnv(ctx, store, "bab", "eleven", map[string]string{"LOG_LEVEL": "debug"}); err != nil {
+ t.Fatal(err)
+ }
+ for _, h := range []string{"awake.example.com", "asleep.example.com"} {
+ rt := execRoute(h, "bab", "eleven")
+ rt.Hibernate = true
+ if err := reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ }
+ cb, ok := rolloutConfigCallbacks(ctx, store, func(h string) bool { return h == "awake.example.com" })
+ if !ok {
+ t.Fatal("refusing mode has no config callbacks")
+ }
+ acks := map[string]appconfig.ConfigAck{}
+ cb.Ack = func(a appconfig.ConfigAck) { acks[a.Host] = a }
+ if err := appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, t.TempDir(), "", cb); err != nil {
+ t.Fatal(err)
+ }
+ awake, ok := acks["awake.example.com"]
+ if !ok {
+ t.Fatalf("no ack for the awake host: %+v", acks)
+ }
+ if awake.Restarted || awake.RestartError != rolloutRestartRefused {
+ t.Fatalf("awake ack = %+v, want Restarted false and RestartError %q", awake, rolloutRestartRefused)
+ }
+ asleep, ok := acks["asleep.example.com"]
+ if !ok {
+ t.Fatalf("no ack for the sleeping host: %+v", acks)
+ }
+ if asleep.RestartError != "" || asleep.Restarted != true || asleep.Hibernating {
+ t.Fatalf("sleeper ack = %+v, want today's: restarted, no error, no marker (the exec pass never sets Hibernating)", asleep)
+ }
+}
+
+// The awake test is the activator's: a managed host it owns. Without an
+// activator, or for a host it does not own, nothing is awake.
+func TestManagedAwakeNeedsTheActivatorToOwnAManagedHost(t *testing.T) {
+ svc, reg, _ := rolloutFixture(t, true)
+ rt := execRoute("m.example.com", "bab", "eleven")
+ rt.Hibernate = true
+ if err := reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ if managedAwake(nil, reg)("m.example.com") {
+ t.Error("awake with no activator")
+ }
+ if managedAwake(svc, reg)("m.example.com") || managedAwake(svc, reg)("gone.example.com") {
+ t.Error("awake though the activator owns nothing")
+ }
+}