| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "github.com/carlosframework/platform/internal/cgroup" |
| 8 | "github.com/carlosframework/platform/internal/instancectl" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | ) |
| 13 | |
| 14 | // TestMemberSafeWakeReason pins the two real 2026-08-09 cases from #131 — |
| 15 | // each needs a different response from the member, and neither may carry |
| 16 | // box filesystem layout. |
| 17 | func TestMemberSafeWakeReason(t *testing.T) { |
| 18 | cases := []struct { |
| 19 | name string |
| 20 | err error |
| 21 | want []string // substrings that must appear |
| 22 | deny []string // substrings that must NOT appear |
| 23 | }{ |
| 24 | { |
| 25 | "stale socket (#129)", |
| 26 | errors.New(`start ledgerpay.bde.oncarlos.com: listen unix /data/exec/ledgerpay.bde.oncarlos.com/inst.sock: bind: address already in use`), |
| 27 | []string{"socket already in use", "previous run"}, |
| 28 | []string{"/data", "bind:"}, |
| 29 | }, |
| 30 | { |
| 31 | "nothing promoted (#130)", |
| 32 | errors.New(`materialize ledgerpay.bde.oncarlos.com: no adoptable head (nothing promoted for its channel, or the head is bake-held)`), |
| 33 | []string{"nothing is promoted", "carlos channels"}, |
| 34 | nil, |
| 35 | }, |
| 36 | { |
| 37 | // skills#29: a static head under an instance route. This one is |
| 38 | // ordered BEFORE the "no adoptable head" case in the renderer, |
| 39 | // and the ordering is the whole point — materialize's sentence |
| 40 | // contains both markers, and matching the generic one first |
| 41 | // would throw away the only diagnosis that names the real |
| 42 | // problem and print the reassuring falsehood instead. |
| 43 | "a static release under an instance route", |
| 44 | errors.New(`materialize techarena.bak.platform.tito.io: the head is a static release: techarena 0.1.0 ` + |
| 45 | `is promoted on edge, but a static site is served straight off the channel pointer and has no ` + |
| 46 | `binary to run, so this route can never wake into it. A static site needs a static route: ` + |
| 47 | "`carlos instances delete --app techarena --host techarena.bak.platform.tito.io`, then " + |
| 48 | "`carlos deploy --app techarena --kind static --host techarena.bak.platform.tito.io <site-dir>`"), |
| 49 | []string{"static release", "--kind static", "0.1.0", "carlos deploy"}, |
| 50 | []string{"nothing is promoted"}, // the contradiction that cost a real session its afternoon |
| 51 | }, |
| 52 | { |
| 53 | "unknown errors keep meaning, lose layout", |
| 54 | errors.New(`seed h: fork/exec /opt/carlos/versions/ledgerpay/1.2.3/ledgerpay: permission denied`), |
| 55 | []string{"wake failed", "permission denied", "ledgerpay"}, |
| 56 | []string{"/opt/carlos"}, |
| 57 | }, |
| 58 | { |
| 59 | // A wake refused while the last failed wake's generation is |
| 60 | // unproven (zero-downtime spec §7.4) reads as the fence's |
| 61 | // existing text: the platform holds the host closed until its |
| 62 | // own work is done. No new member-facing text. |
| 63 | "a wake refused over an unproven generation", |
| 64 | fmt.Errorf("h.example.com: refusing to wake: %w", errWakeUnproven), |
| 65 | []string{memberSafeWakeReason(ErrFenced)}, |
| 66 | []string{"generation", "proven"}, |
| 67 | }, |
| 68 | { |
| 69 | // The first failure itself (final review F5): a contained start |
| 70 | // that failed its health check and whose generation kill was not |
| 71 | // proven. Its sentinel's text is operator detail; the member gets |
| 72 | // the same fence text. With generations off too (containment can |
| 73 | // be on without them). |
| 74 | "a start whose generation is unproven", |
| 75 | fmt.Errorf("start h.example.com: health: no answer: %w", fmt.Errorf("kill g7: %w", errGenUnproven)), |
| 76 | []string{memberSafeWakeReason(ErrFenced)}, |
| 77 | []string{"generation", "proven", "health"}, |
| 78 | }, |
| 79 | { |
| 80 | "a nested start whose generation is unproven", |
| 81 | fmt.Errorf("start h.example.com: %w", instancectl.ErrGenUnproven), |
| 82 | []string{memberSafeWakeReason(ErrFenced)}, |
| 83 | []string{"generation", "unproven", "instance control"}, |
| 84 | }, |
| 85 | } |
| 86 | for _, tc := range cases { |
| 87 | t.Run(tc.name, func(t *testing.T) { |
| 88 | got := memberSafeWakeReason(tc.err) |
| 89 | for _, w := range tc.want { |
| 90 | if !strings.Contains(got, w) { |
| 91 | t.Errorf("missing %q in %q", w, got) |
| 92 | } |
| 93 | } |
| 94 | for _, d := range tc.deny { |
| 95 | if strings.Contains(got, d) { |
| 96 | t.Errorf("leaked %q in %q", d, got) |
| 97 | } |
| 98 | } |
| 99 | }) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | // eventRec and fakeSink are the package's EventSink fixture. |
| 104 | type eventRec struct{ acct, app, host, level, msg string } |
| 105 | |
| 106 | type fakeSink struct { |
| 107 | apps []eventRec |
| 108 | events []eventRec |
| 109 | } |
| 110 | |
| 111 | func (f *fakeSink) App(acct, app, host, line string) { |
| 112 | f.apps = append(f.apps, eventRec{acct, app, host, "", line}) |
| 113 | } |
| 114 | func (f *fakeSink) Event(acct, app, host, level, msg string) { |
| 115 | f.events = append(f.events, eventRec{acct, app, host, level, msg}) |
| 116 | } |
| 117 | |
| 118 | // TestWakeOutcomesReachTheSink: the activator's wake path publishes exactly |
| 119 | // one member-facing event per outcome — the #131 fact, at its source. The |
| 120 | // route carries app+account context so the sink can key the record. |
| 121 | func TestWakeOutcomesReachTheSink(t *testing.T) { |
| 122 | t.Parallel() |
| 123 | // Failure: Start refuses, the event is the member-safe rendering. |
| 124 | a, b, _, _ := testActivator(t, time.Hour) |
| 125 | sink := &fakeSink{} |
| 126 | a.Logs = sink |
| 127 | b.startErr = errors.New("listen unix /data/exec/h/inst.sock: bind: address already in use") |
| 128 | rt := testRoute("ledgerpay.bde.oncarlos.com") |
| 129 | rt.App = "ledgerpay" |
| 130 | rt.Slug = "" |
| 131 | if _, err := a.beginRequest(context.Background(), rt); err == nil { |
| 132 | t.Fatal("wake should have failed") |
| 133 | } |
| 134 | var wakeFails []eventRec |
| 135 | for _, e := range sink.events { |
| 136 | if strings.Contains(e.msg, "wake failed") { |
| 137 | wakeFails = append(wakeFails, e) |
| 138 | } |
| 139 | } |
| 140 | if len(wakeFails) != 1 { |
| 141 | t.Fatalf("wake-failure events: %d (%v)", len(wakeFails), sink.events) |
| 142 | } |
| 143 | ev := wakeFails[0] |
| 144 | if ev.level != "error" || ev.host != "ledgerpay.bde.oncarlos.com" || ev.app != "ledgerpay" { |
| 145 | t.Fatalf("event mis-tagged: %+v", ev) |
| 146 | } |
| 147 | if strings.Contains(ev.msg, "/data") || !strings.Contains(ev.msg, "socket already in use") { |
| 148 | t.Fatalf("event not member-safe: %q", ev.msg) |
| 149 | } |
| 150 | |
| 151 | // Success: a woke event, tagged the same way. |
| 152 | a2, _, _, _ := testActivator(t, time.Hour) |
| 153 | sink2 := &fakeSink{} |
| 154 | a2.Logs = sink2 |
| 155 | rel, err := a2.beginRequest(context.Background(), rt) |
| 156 | if err != nil { |
| 157 | t.Fatalf("wake failed: %v", err) |
| 158 | } |
| 159 | rel() |
| 160 | var woke bool |
| 161 | for _, e := range sink2.events { |
| 162 | if strings.Contains(e.msg, "instance woke") && e.level == "info" { |
| 163 | woke = true |
| 164 | } |
| 165 | } |
| 166 | if !woke { |
| 167 | t.Fatalf("no woke event: %+v", sink2.events) |
| 168 | } |
| 169 | } |
| 170 | |
| 171 | // A claims-on wake on the closed launch fails before its exec hook, and |
| 172 | // startAdmitted wraps that in errClaimsAdmission so the one classifier |
| 173 | // reads it as the admission's (Task 10). The wrapper is internal: the |
| 174 | // member's event renders exactly as the bare cause did on main, whatever |
| 175 | // the flag, alone or joined with FailedAdmission's error as doClaimWake |
| 176 | // returns it. No "claims admission" text reaches a member. |
| 177 | func TestMemberSafeWakeReasonHidesTheClaimsAdmissionWrapper(t *testing.T) { |
| 178 | recovery := errors.New("recovery: release the admission") |
| 179 | for _, tc := range []struct { |
| 180 | name string |
| 181 | wrapped, bare error |
| 182 | }{ |
| 183 | {"the closed launch", claimAdmission(ErrClaimIsolationUnavailable, false), ErrClaimIsolationUnavailable}, |
| 184 | {"the closed launch, wrapped by the launcher", claimAdmission(fmt.Errorf("start main: %w", ErrClaimIsolationUnavailable), false), |
| 185 | fmt.Errorf("start main: %w", ErrClaimIsolationUnavailable)}, |
| 186 | {"joined as doClaimWake returns it", errors.Join(claimAdmission(ErrClaimIsolationUnavailable, false), nil), |
| 187 | errors.Join(ErrClaimIsolationUnavailable, nil)}, |
| 188 | {"joined with a recovery error", errors.Join(claimAdmission(ErrClaimIsolationUnavailable, false), recovery), |
| 189 | errors.Join(ErrClaimIsolationUnavailable, recovery)}, |
| 190 | } { |
| 191 | got, want := memberSafeWakeReason(tc.wrapped), memberSafeWakeReason(tc.bare) |
| 192 | if got != want { |
| 193 | t.Errorf("%s: %q, want main's %q", tc.name, got, want) |
| 194 | } |
| 195 | if strings.Contains(got, "claims admission") { |
| 196 | t.Errorf("%s: the admission wrapper reached a member: %q", tc.name, got) |
| 197 | } |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | // Amadan #272: a wake refused because the host's parent cgroup holds a |
| 202 | // process directly reads to a member as the maintenance fence's approved |
| 203 | // text, not as cgroup detail. |
| 204 | func TestMemberSafeWakeReasonHidesALegacyParentRefusal(t *testing.T) { |
| 205 | err := fmt.Errorf("h.example.com: %w: h.example.com holds 2 process(es)", cgroup.ErrLegacyParentPopulated) |
| 206 | got := memberSafeWakeReason(err) |
| 207 | if got != memberSafeWakeReason(ErrFenced) || strings.Contains(got, "cgroup") || strings.Contains(got, "process") { |
| 208 | t.Errorf("member reason = %q, want the fence text alone", got) |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | // Amadan #272, Task 19: a wake refused because its cgroup took cgroup.kill |
| 213 | // and could not be made fresh reads to a member as the maintenance fence's |
| 214 | // approved text, not as cgroup detail. Nothing of the build ran, so the |
| 215 | // wake is never one that may have started (startMayHaveRun). |
| 216 | func TestMemberSafeWakeReasonHidesAKilledCgroupRefusal(t *testing.T) { |
| 217 | err := fmt.Errorf("%w: /sys/fs/cgroup/x/carlos-exec@h.example.com: device or resource busy", cgroup.ErrKilledCgroupKept) |
| 218 | got := memberSafeWakeReason(err) |
| 219 | if got != memberSafeWakeReason(ErrFenced) || strings.Contains(got, "cgroup") || strings.Contains(got, "busy") { |
| 220 | t.Errorf("member reason = %q, want the fence text alone", got) |
| 221 | } |
| 222 | if startMayHaveRun(nil, err) { |
| 223 | t.Error("a start refused for a killed cgroup counted as one that may have run") |
| 224 | } |
| 225 | } |
| 226 | |
| 227 | // Task 7 (readiness round 2): a wake refused because its generation's limits |
| 228 | // could not be applied reads to a member as the maintenance fence's approved |
| 229 | // text, not as cgroup detail. Nothing of the build ran. |
| 230 | func TestMemberSafeWakeReasonHidesALimitNotAppliedRefusal(t *testing.T) { |
| 231 | for _, err := range []error{ |
| 232 | fmt.Errorf("h.example.com: %w: g2: memory.max: open /sys/fs/cgroup/x/carlos-exec@h.example.com/g2/memory.max: permission denied", cgroup.ErrLimitNotApplied), |
| 233 | fmt.Errorf("%w: h.example.com: the placement shim is not available", cgroup.ErrLimitNotApplied), |
| 234 | } { |
| 235 | got := memberSafeWakeReason(err) |
| 236 | if got != memberSafeWakeReason(ErrFenced) || strings.Contains(got, "cgroup") || strings.Contains(got, "memory") { |
| 237 | t.Errorf("member reason = %q, want the fence text alone", got) |
| 238 | } |
| 239 | if startMayHaveRun(nil, err) { |
| 240 | t.Error("a start refused for its limits counted as one that may have run") |
| 241 | } |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | // Whole-branch review round 1 of zdd readiness 3, finding 3. A start whose |
| 246 | // exec.CommandContext context ends while the cgroup manager's shim is being |
| 247 | // placed answers the context's own error, bare, as exec.Cmd.Start answers |
| 248 | // an ended context (cgroup's startIn; its unit and kernel tests pin that |
| 249 | // shape). The only spawn built with a context is the seed, so this is that |
| 250 | // error as the seed wraps it and the wake event renders it: the classifier |
| 251 | // reads a cancellation as the caller's own end, and the member reads what a |
| 252 | // direct start's ended context always read, with no placement, shim or |
| 253 | // cgroup words (those stay in the agent's log line) and no new text. |
| 254 | func TestCancelledPlacementReadsAsTheCallersOwnEnd(t *testing.T) { |
| 255 | const host = "h.example.com" |
| 256 | cancelled := fmt.Errorf("seeding %s: %w", host, context.Canceled) |
| 257 | timedOut := fmt.Errorf("seeding %s: %w", host, fmt.Errorf("timed out after %s and was killed: %w", seedTimeout, context.DeadlineExceeded)) |
| 258 | if got := classifyStep(nil, cancelled); got != stepCancelled { |
| 259 | t.Errorf("classifyStep(cancelled placement) = %v, want stepCancelled", got) |
| 260 | } |
| 261 | if got, want := classifyStep(nil, timedOut), classifyStep(nil, context.DeadlineExceeded); got != want { |
| 262 | t.Errorf("classifyStep(timed-out placement) = %v, want %v, as for a direct start's deadline", got, want) |
| 263 | } |
| 264 | for err, want := range map[error]string{ |
| 265 | cancelled: "wake failed: seeding h.example.com: context canceled", |
| 266 | timedOut: "wake failed: seeding h.example.com: timed out after " + seedTimeout.String() + " and was killed: context deadline exceeded", |
| 267 | } { |
| 268 | got := memberSafeWakeReason(err) |
| 269 | if got != want { |
| 270 | t.Errorf("member reason = %q, want %q", got, want) |
| 271 | } |
| 272 | for _, word := range []string{"placement", "shim", "cgroup"} { |
| 273 | if strings.Contains(got, word) { |
| 274 | t.Errorf("member reason %q carries %q", got, word) |
| 275 | } |
| 276 | } |
| 277 | if startMayHaveRun(nil, err) { |
| 278 | t.Errorf("a start whose context ended during placement counted as one that may have run: %v", err) |
| 279 | } |
| 280 | } |
| 281 | } |
| 282 | |
amadan