| 1 | // The error shipping loop — the box-side pump of the Errors tab (spec |
| 2 | // 2026-09-07-errors-and-analytics-design.md §3.2), shaped exactly like |
| 3 | // logshipLoop: the edge and the activator coalesce into one apperr.Sink; |
| 4 | // every tick this drains it and writes one immutable chunk per app under |
| 5 | // apps/<sqid>/<app>/errors/<box>/<hour>/; on each hour boundary it prunes |
| 6 | // its own subtree with the log retention (§3.6: errors follow log |
| 7 | // retention, age and byte cap). |
| 8 | package host |
| 9 | |
| 10 | import ( |
| 11 | "context" |
| 12 | "errors" |
| 13 | "log" |
| 14 | "os" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/apperr" |
| 18 | "github.com/carlosframework/platform/internal/applog" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | ) |
| 21 | |
| 22 | const errshipTick = 10 * time.Second |
| 23 | |
| 24 | func errshipLoop(sink *apperr.Sink, box string) { |
| 25 | defer func() { |
| 26 | if r := recover(); r != nil { |
| 27 | log.Printf("agent: errship loop panic, error shipping stopped for this process: %v", r) |
| 28 | } |
| 29 | }() |
| 30 | ctx := context.Background() |
| 31 | s, target, err := openStatusStore(ctx, os.Getenv) |
| 32 | if err != nil { |
| 33 | if errors.Is(err, errNoStatusStore) { |
| 34 | log.Printf("agent: errship loop disabled: neither CARLOS_STATUS_BUCKET nor CARLOS_DEPLOYMENT_BUCKET is set") |
| 35 | } else { |
| 36 | log.Printf("agent: errship loop disabled: %v", err) |
| 37 | } |
| 38 | return |
| 39 | } |
| 40 | log.Printf("agent: errship loop armed (10s ticks), publishing to %s", target) |
| 41 | |
| 42 | pending := make(map[applog.AppKey][]apperr.Record) |
| 43 | pruned := "" |
| 44 | ticker := time.NewTicker(errshipTick) |
| 45 | defer ticker.Stop() |
| 46 | for range ticker.C { |
| 47 | errshipGather(sink, pending) |
| 48 | errshipFlush(ctx, s, pending) |
| 49 | if hour := applog.HourOf(time.Now()); hour != pruned { |
| 50 | for _, k := range errshipKnownApps(pending, sink) { |
| 51 | chunkPrune(ctx, s, applog.FamilyErrors, box, k, time.Now()) |
| 52 | } |
| 53 | pruned = hour |
| 54 | } |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | // errshipGather drains the sink into pending under the same per-app cap |
| 59 | // logship uses — oldest first. |
| 60 | func errshipGather(sink *apperr.Sink, pending map[applog.AppKey][]apperr.Record) { |
| 61 | for k, recs := range sink.Drain() { |
| 62 | merged := append(pending[k], recs...) |
| 63 | if over := len(merged) - logshipMaxPending; over > 0 { |
| 64 | merged = merged[over:] |
| 65 | } |
| 66 | pending[k] = merged |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | // errshipFlush writes one create-only chunk per pending app and forgets |
| 71 | // what landed; a failed app keeps its records for the next round. |
| 72 | func errshipFlush(ctx context.Context, s release.Store, pending map[applog.AppKey][]apperr.Record) { |
| 73 | for k, recs := range pending { |
| 74 | if len(recs) == 0 { |
| 75 | delete(pending, k) |
| 76 | continue |
| 77 | } |
| 78 | body, err := apperr.EncodeChunk(recs) |
| 79 | if err != nil { |
| 80 | log.Printf("agent: errship encode %s/%s: %v (dropping %d records)", k.Acct, k.App, err, len(recs)) |
| 81 | delete(pending, k) |
| 82 | continue |
| 83 | } |
| 84 | key, err := apperr.ChunkKey(k.Acct, k.App, recs[0].Box, apperr.HourOf(recs[0].TS), recs[0].Seq, len(body)) |
| 85 | if err != nil { |
| 86 | unshippable(applog.FamilyErrors, k, err) |
| 87 | delete(pending, k) |
| 88 | continue |
| 89 | } |
| 90 | if _, err := s.Put(ctx, key, body, ""); err != nil { |
| 91 | log.Printf("agent: errship flush %s/%s: %v (%d records held)", k.Acct, k.App, err, len(recs)) |
| 92 | continue |
| 93 | } |
| 94 | delete(pending, k) |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | func errshipKnownApps(pending map[applog.AppKey][]apperr.Record, sink *apperr.Sink) []applog.AppKey { |
| 99 | seen := make(map[applog.AppKey]bool) |
| 100 | var out []applog.AppKey |
| 101 | for k := range pending { |
| 102 | if !seen[k] { |
| 103 | seen[k] = true |
| 104 | out = append(out, k) |
| 105 | } |
| 106 | } |
| 107 | for _, k := range sink.Known() { |
| 108 | if !seen[k] { |
| 109 | seen[k] = true |
| 110 | out = append(out, k) |
| 111 | } |
| 112 | } |
| 113 | return out |
| 114 | } |
| 115 | |
amadan