amadan
zdd plan 3 final fix: unproven-start sentinels read as the fence to members; Shutdown's flag-off wait recorded (F5, T13)
memberSafeWakeReason maps errGenUnproven and instancectl.ErrGenUnproven, not only errWakeUnproven, to the maintenance fence's existing text, so a contained start that failed health with its kill unproven (generations on or off) puts no generation detail in the member's event. wake() logs that detail for the operator. The Shutdown comment no longer claims the retained-wake wait is empty with generations off: the leaderLives retention fills it on any box, so flag-off Shutdown waits there too, bounded by ctx.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
3 files changed,
+40
−8
internal/activator/activator.go+14 −3internal/activator/logsink.go+8 −5internal/activator/logsink_test.go+18 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go| index cf344a66a..96bee789a 100644 |
| --- a/internal/activator/activator.go |
| +++ b/internal/activator/activator.go |
| @@ -1206,6 +1206,11 @@ func (a *Service) wake(e *actEntry) { |
| // the route and the whole error are in hand — the journal line below the |
| // backends already write stays exactly as it was. |
| if err != nil { |
| + if genStartUnproven(err) { |
| + // The member's event reads as the fence (memberSafeWakeReason); |
| + // the generation detail is the operator's. |
| + log.Printf("activator: %s: wake failed and its generation is not proven stopped: %v", e.host, err) |
| + } |
| if msg, ok := a.wakeFailureEvent(e, err); ok { |
| emitEvent(a.Logs, e.route, "error", msg) |
| } |
| @@ -3650,9 +3655,15 @@ func (a *Service) Shutdown(ctx context.Context) { |
| // reaper (reapUnprovenWakesLocked, inside finishPendingStops) retries |
| // each one's stop and proof and gives its lease back once it holds. A |
| // wake still unproven when the budget runs out keeps its lease; the |
| - // lease's TTL frees it. With generations off this map fills only from |
| - // GenContainment's sentinels (genStartUnproven), so it is empty on a box |
| - // without containment and this returns at once. |
| + // lease's TTL frees it. |
| + // |
| + // With generations off the map fills too, so flag-off Shutdown waits |
| + // here as well, bounded by ctx: from GenContainment's sentinels |
| + // (genStartUnproven), and, on any box, from a failed wake whose leader |
| + // survived its stop (cleanupFailedWake's leaderLives, the fail-closed |
| + // exception to dormancy from the task 2 fix). That flag-off retention |
| + // proves only the leader's exit, not its descendants (final review F6, |
| + // deferred). With neither, the map is empty and this returns at once. |
| for { |
| a.mu.Lock() |
| left := len(a.unprovenWakes) |
diff --git a/internal/activator/logsink.go b/internal/activator/logsink.go| index e025bb6df..3562ac961 100644 |
| --- a/internal/activator/logsink.go |
| +++ b/internal/activator/logsink.go |
| @@ -48,12 +48,15 @@ var absPathToken = regexp.MustCompile(`(^|[\s"(=])(/[\w@.\-/]+)`) |
| // text itself originates in our own code (backends, materialize, lease) — |
| // this is a rendering pass, not a secrecy boundary. |
| func memberSafeWakeReason(err error) string { |
| - if errors.Is(err, errWakeUnproven) { |
| + if errors.Is(err, errWakeUnproven) || genStartUnproven(err) { |
| // A wake refused while the host's last failed wake is not proven |
| - // stopped (zero-downtime spec §7.4). Members get the maintenance |
| - // fence's existing text, the state that is like it: the platform |
| - // holds the host closed until its own work is done, and it reopens |
| - // by itself. Internal detail about generations stays in the log. |
| + // stopped (zero-downtime spec §7.4), or that failed wake itself: a |
| + // start whose generation kill was not proven (errGenUnproven, |
| + // instancectl.ErrGenUnproven), which containment produces with |
| + // generations on or off. Members get the maintenance fence's |
| + // existing text, the state that is like it: the platform holds the |
| + // host closed until its own work is done, and it reopens by itself. |
| + // The generation detail stays in the operator log (wake). |
| err = ErrFenced |
| } |
| s := err.Error() |
diff --git a/internal/activator/logsink_test.go b/internal/activator/logsink_test.go| index f1649cae0..d1908d5b8 100644 |
| --- a/internal/activator/logsink_test.go |
| +++ b/internal/activator/logsink_test.go |
| @@ -4,6 +4,7 @@ import ( |
| "context" |
| "errors" |
| "fmt" |
| + "github.com/carlosframework/platform/internal/instancectl" |
| "strings" |
| "testing" |
| "time" |
| @@ -63,6 +64,23 @@ func TestMemberSafeWakeReason(t *testing.T) { |
| []string{memberSafeWakeReason(ErrFenced)}, |
| []string{"generation", "proven"}, |
| }, |
| + { |
| + // The first failure itself (final review F5): a contained start |
| + // that failed its health check and whose generation kill was not |
| + // proven. Its sentinel's text is operator detail; the member gets |
| + // the same fence text. With generations off too (containment can |
| + // be on without them). |
| + "a start whose generation is unproven", |
| + fmt.Errorf("start h.example.com: health: no answer: %w", fmt.Errorf("kill g7: %w", errGenUnproven)), |
| + []string{memberSafeWakeReason(ErrFenced)}, |
| + []string{"generation", "proven", "health"}, |
| + }, |
| + { |
| + "a nested start whose generation is unproven", |
| + fmt.Errorf("start h.example.com: %w", instancectl.ErrGenUnproven), |
| + []string{memberSafeWakeReason(ErrFenced)}, |
| + []string{"generation", "unproven", "instance control"}, |
| + }, |
| } |
| for _, tc := range cases { |
| t.Run(tc.name, func(t *testing.T) { |