| 1 | // The activator's half of `carlos logs` (spec 2026-08-10-carlos-logs-design |
| 2 | // §5): the seam the agent injects its applog sink through, and the |
| 3 | // member-safe rendering of wake failures. The reason a wake failed is the |
| 4 | // whole diagnosis a member gets for a 503 (#131), and it is composed HERE, |
| 5 | // at the emit site — never relayed from the journal, and never carrying box |
| 6 | // filesystem layout. |
| 7 | package activator |
| 8 | |
| 9 | import ( |
| 10 | "errors" |
| 11 | "fmt" |
| 12 | "regexp" |
| 13 | "strings" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/cgroup" |
| 16 | "github.com/carlosframework/platform/internal/registry" |
| 17 | "github.com/carlosframework/platform/internal/release" |
| 18 | "github.com/carlosframework/platform/internal/updatewords" |
| 19 | ) |
| 20 | |
| 21 | // EventSink is where the activator publishes member-visible log lines. |
| 22 | // applog.Sink satisfies it; the interface lives here so this package |
| 23 | // depends on the contract, not the buffer. Nil disables everything — |
| 24 | // tests and non-shipping deployments change nothing. |
| 25 | type EventSink interface { |
| 26 | // App records one line of the app's own output. |
| 27 | App(acct, app, host, line string) |
| 28 | // Event records one platform event with member-safe text. |
| 29 | Event(acct, app, host, level, msg string) |
| 30 | } |
| 31 | |
| 32 | // emitEvent publishes one platform event for rt's app, tolerating a nil |
| 33 | // sink. level is an applog.Level* value ("info"/"warn"/"error" — string |
| 34 | // here to keep the dependency one-way). |
| 35 | func emitEvent(sink EventSink, rt registry.Route, level, msg string) { |
| 36 | if sink == nil { |
| 37 | return |
| 38 | } |
| 39 | sink.Event(release.RouteAccount(rt), rt.App, rt.Host, level, msg) |
| 40 | } |
| 41 | |
| 42 | // absPathToken matches an absolute-path token: box filesystem layout, |
| 43 | // which a member-safe message must not carry (#131). Replacement keeps the |
| 44 | // base name — "…/ledgerpay.bde.oncarlos.com.sock" still tells the member WHICH |
| 45 | // socket without telling them where the box keeps it. |
| 46 | var absPathToken = regexp.MustCompile(`(^|[\s"(=])(/[\w@.\-/]+)`) |
| 47 | |
| 48 | // memberSafeWakeReason renders a wake error for the member-facing platform |
| 49 | // stream. Known failure shapes get a diagnosis in the member's terms; the |
| 50 | // rest pass through with absolute paths reduced to base names. The error |
| 51 | // text itself originates in our own code (backends, materialize, lease) — |
| 52 | // this is a rendering pass, not a secrecy boundary. |
| 53 | func memberSafeWakeReason(err error) string { |
| 54 | // The seed fallback's refusals are composed for the member (SD-N14 to |
| 55 | // SD-N19) and pass through whole, without the wrapping around them. |
| 56 | var seed *SeedPlacementError |
| 57 | if errors.As(err, &seed) { |
| 58 | return "wake failed: " + seed.Sentence |
| 59 | } |
| 60 | // The held wake check's refusals (settings spec §4.3 rule 3; ruling |
| 61 | // R-WAKEWORDS). Unknown and failed need the member's action, so they get |
| 62 | // the settings words (approved copy, settings plan 4). A wait |
| 63 | // for root that ran out, or a repoint after it, is the platform's own |
| 64 | // work and reopens by itself: the fence's existing wording. |
| 65 | if errors.Is(err, errBuildUnknown) { |
| 66 | return updatewords.Text("settings.box.reason.unknown") |
| 67 | } |
| 68 | var failed *targetFailedError |
| 69 | if errors.As(err, &failed) { |
| 70 | return fmt.Sprintf(updatewords.Text("settings.box.reason.failed"), failed.build) |
| 71 | } |
| 72 | if errors.Is(err, errStaleLink) { |
| 73 | err = ErrFenced |
| 74 | } |
| 75 | if errors.Is(err, errStagedBuildNotPublished) { |
| 76 | // A staged instance wake refused because its build could not be |
| 77 | // published first (final review F9, flag on only). The platform |
| 78 | // holds the host closed until its own write succeeds (the sweep |
| 79 | // retries it) and then it reopens by itself, which is the |
| 80 | // maintenance fence's existing wording; the write error is the |
| 81 | // operator's (persistGenerations logs it). |
| 82 | err = ErrFenced |
| 83 | } |
| 84 | if errors.Is(err, errBuildStateUnwritten) { |
| 85 | // A spawn refused because its build state could not be written |
| 86 | // first (settings spec §4.2; ruling R-WAKEWORDS): the platform holds |
| 87 | // the host until its own write succeeds, the fence's existing |
| 88 | // wording. The write error is the operator's (recordBuild logs it). |
| 89 | err = ErrFenced |
| 90 | } |
| 91 | if errors.Is(err, cgroup.ErrLegacyParentPopulated) { |
| 92 | // A generation start refused because the host's parent cgroup still |
| 93 | // holds a process directly (amadan #272). It is the box's state to |
| 94 | // clear: the platform does not lift it by itself; the process must |
| 95 | // exit or be stopped (or the flag go off). The cgroup detail (the |
| 96 | // host, the count) is the operator's: the agent logs it. The member |
| 97 | // gets the maintenance fence's existing text, the box's condition |
| 98 | // in the words the platform already uses for it. |
| 99 | err = ErrFenced |
| 100 | } |
| 101 | if errors.Is(err, cgroup.ErrLimitNotApplied) { |
| 102 | // A generation start refused because a limit it would run under |
| 103 | // could not be applied, or it could only be started outside its |
| 104 | // cgroup and moved after (zdd readiness round 2, Task 7). Like the |
| 105 | // populated parent above it is the box's condition, and the cgroup |
| 106 | // detail is the operator's (the manager logs it), so the member |
| 107 | // gets the maintenance fence's existing text. |
| 108 | err = ErrFenced |
| 109 | } |
| 110 | if errors.Is(err, cgroup.ErrKilledCgroupKept) { |
| 111 | // A start refused because the cgroup it would spawn into took |
| 112 | // cgroup.kill and could not be made fresh (amadan #272, Task 19). |
| 113 | // Like the populated parent above it is the box's condition, and the |
| 114 | // cgroup detail is the operator's (the manager logs it), so the |
| 115 | // member gets the maintenance fence's existing text. |
| 116 | err = ErrFenced |
| 117 | } |
| 118 | if errors.Is(err, errHostUnproven) { |
| 119 | // A flag-off wake refused because its host cgroup holds processes |
| 120 | // not proven gone (Task 8 fix round 1, finding 4): the platform holds |
| 121 | // the host closed until its own proof holds, and it reopens by |
| 122 | // itself. The fence's existing text; the cgroup detail is the |
| 123 | // operator's (the wake logs it). |
| 124 | err = ErrFenced |
| 125 | } |
| 126 | if errors.Is(err, errWakeUnproven) || genStartUnproven(err) || errors.Is(err, errOutcomeUndelivered) { |
| 127 | // A wake refused while the host's last failed wake is not proven |
| 128 | // stopped (zero-downtime spec §7.4), or that failed wake itself: a |
| 129 | // start whose generation kill was not proven (errGenUnproven, |
| 130 | // instancectl.ErrGenUnproven), which containment produces with |
| 131 | // generations on or off. Members get the maintenance fence's |
| 132 | // existing text, the state that is like it: the platform holds the |
| 133 | // host closed until its own work is done, and it reopens by itself. |
| 134 | // The generation detail stays in the operator log (wake). The same |
| 135 | // for a wake refused while an earlier planned wake's outcome is |
| 136 | // still being delivered (errOutcomeUndelivered, final review wave 2 |
| 137 | // N1). |
| 138 | err = ErrFenced |
| 139 | } |
| 140 | if errors.Is(err, ErrBuildNotPublished) { |
| 141 | // A worker wake refused because its build could not be published |
| 142 | // before the exec (plan 4's ErrBuildNotPublished, flag on or off). |
| 143 | // The member gets a reason in their terms; the write error is the |
| 144 | // operator's (the sweep retries the write). errStagedBuildNotPublished |
| 145 | // is a distinct sentinel and was mapped to the fence above. |
| 146 | return "wake failed: the box could not record which build this instance runs, so it did not start it. the next request tries again" |
| 147 | } |
| 148 | s := err.Error() |
| 149 | if errors.Is(err, errClaimsAdmission) { |
| 150 | // claimAdmission's wrapper is the classifier's, not the member's |
| 151 | // (zero-downtime plan 3, Task 10): the event reads exactly as its |
| 152 | // cause did, alone or joined. Every other mapping is unchanged. |
| 153 | s = strings.ReplaceAll(s, claimAdmissionPrefix, "") |
| 154 | } |
| 155 | switch { |
| 156 | case strings.Contains(s, "address already in use"): |
| 157 | // The stale-socket wake trap (#129): every later wake fails until |
| 158 | // the leftover socket goes away. Say that, not the bind syscall. |
| 159 | return "wake failed: instance socket already in use — a previous run's socket was left behind (this wake will keep failing until the platform clears it)" |
| 160 | case strings.Contains(s, release.StaticHeadUnderInstance): |
| 161 | // The one wake failure `carlos channels` appears to CONTRADICT: the |
| 162 | // channel does have a head, it is just a static one under a route |
| 163 | // that execs a binary (skills#29). materialize's sentence already |
| 164 | // names the app, the version, the channel and both commands out of |
| 165 | // it — it is composed for a member, so it is passed through whole |
| 166 | // rather than summarised into something less actionable. |
| 167 | return "wake failed: " + RedactPaths(s) |
| 168 | case strings.Contains(s, "no adoptable head"): |
| 169 | return "wake failed: nothing is promoted for this instance's channel (or the head is still bake-held) — promote a build, or check `carlos channels`" |
| 170 | } |
| 171 | return "wake failed: " + RedactPaths(s) |
| 172 | } |
| 173 | |
| 174 | // RedactPaths reduces absolute-path tokens to their base names. Exported |
| 175 | // for the host package's schedule wiring, whose token and delivery errors |
| 176 | // reach the bucket status object and `carlos schedule ls` (#131's rule: |
| 177 | // no box filesystem layout in a member-facing string). |
| 178 | func RedactPaths(s string) string { |
| 179 | return absPathToken.ReplaceAllStringFunc(s, func(m string) string { |
| 180 | i := strings.Index(m, "/") |
| 181 | prefix, path := m[:i], m[i:] |
| 182 | if j := strings.LastIndexByte(path, '/'); j >= 0 && j+1 < len(path) { |
| 183 | return prefix + path[j+1:] |
| 184 | } |
| 185 | return prefix + path |
| 186 | }) |
| 187 | } |
| 188 | |
amadan