amadan
slice A only: take the replica-freshness work back out (#284)
Summary
Removes the slice B pieces (replica alert rule, status route_replicas, lease.Replicator.Level, activator.ReplicaLevels, the console's replication column). What stays is slice A: the memory cap sized from the database count, restart backoff, the restarts alert rule with its box sentences, and the console's alert detail for it.
Agent notes
The removed work is on branch replica-freshness-284 at b110af29, waiting on a revised design (amadan #290). User-visible strings here are drafts until Paul's copy review; the plan14 census overlay is still owed.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
18 files changed,
+87
−761
apimode.go+0 −11docs/site/cli.md+0 −7instances_cli.go+3 −24instances_cli_test.go+0 −63instances_e2e_test.go+0 −3internal/activator/replicalevel.go+0 −70internal/activator/replicalevel_test.go+0 −60internal/console/alertdetail_test.go+76 −0internal/console/alertwatch.go+0 −4internal/console/apiinstances.go+2 −83internal/console/overview.go+0 −34internal/console/replicafresh_test.go+0 −261internal/console/server.go+0 −9internal/console/templates/app_instances.html+2 −3internal/lease/lease.go+0 −22internal/metrics/alert.go+2 −35internal/metrics/alert_test.go+0 −38internal/status/snapshot.go+2 −34
CI cancelled — superseded by a newer push — run details
CI log
=== gate === cd launchctx && go vet ./... && go test ./... -count=1 ok amadan.net/carlos/platform.git/launchctx 0.011s ? amadan.net/carlos/platform.git/launchctx/controlname [no test files] ? amadan.net/carlos/platform.git/launchctx/wireid [no test files] go vet ./... gofmt -l .: clean go test ./... -count=1 -timeout 30m amadan: job cancelled — a newer push superseded it
diff --git a/apimode.go b/apimode.go| index dd5692eb1..00177a653 100644 |
| --- a/apimode.go |
| +++ b/apimode.go |
| @@ -3224,13 +3224,6 @@ func (s *apiSession) instancesEnable(ctx context.Context, a apiAcct, app string, |
| // apiInstanceRecord mirrors provision.Record's wire shape — redeclared here |
| // rather than imported because this client renders records, never writes |
| // them, and the fields it renders are the contract worth pinning. |
| -// apiReplica is the list's replica object. State is "level", "behind", |
| -// "late", "parked" or "unknown"; BehindSecs is set for "behind" and "late". |
| -type apiReplica struct { |
| - State string `json:"state"` |
| - BehindSecs *int64 `json:"behind_secs,omitempty"` |
| -} |
| - |
| type apiInstanceRecord struct { |
| Host string `json:"host"` |
| Slug string `json:"slug"` |
| @@ -3296,10 +3289,6 @@ type apiInstanceRecord struct { |
| // measured it (instanceListRecord.DBBytes). nil when unknown, or from a |
| // console that predates the field — the DB column prints a dash. |
| DBBytes *int64 `json:"db_bytes,omitempty"` |
| - // Replica is where the instance's off-box replica stands as its box last |
| - // reported it (instanceListRecord.Replica). nil when unknown, or from a |
| - // console that predates the field: the REPLICA column prints a dash. |
| - Replica *apiReplica `json:"replica,omitempty"` |
| // PrivateStoreBucket is the bucket the CREATE door provisioned for this |
| // instance under the app's private-storage policy (issue #412). Set on |
| // exactly one response — the create that minted it — and empty |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index 30f7ef9c0..e2bc11208 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -734,13 +734,6 @@ every few minutes, and asleep instances are measured too. A dash means no box |
| has measured it: the instance is not running anywhere yet, it has not created |
| its database, or it keeps its data somewhere CARLOS does not manage. |
| -The `REPLICA` column says whether each instance's off-box copy of that |
| -database is keeping up. `ok` means it is, or the instance is asleep and |
| -finished its last sync. A time such as `2h10m behind` means the copy has |
| -trailed the database for that long, past its sync interval and a grace of |
| -fifteen minutes. A dash means the instance's box has not said, which is not |
| -the same as `ok`. |
| - |
| `carlos instances list` shows two builds for each instance: the one installed |
| on its box, and the one serving its requests. They differ while a new build |
| rolls out, while an asleep instance waits for its next request, and after a |
diff --git a/instances_cli.go b/instances_cli.go| index 7b5380651..a147fc534 100644 |
| --- a/instances_cli.go |
| +++ b/instances_cli.go |
| @@ -32,7 +32,6 @@ import ( |
| "net/url" |
| "os" |
| "strings" |
| - "time" |
| "github.com/carlosframework/platform/internal/appconfig" |
| "github.com/carlosframework/platform/internal/provision" |
| @@ -431,28 +430,8 @@ func upstreamsCell(ups []apiUpstreamState) string { |
| return fmt.Sprintf("%d/%d", up, len(ups)) |
| } |
| -// replicaCell is the REPLICA column. Only a replica that is late is worth |
| -// words: a level, parked or behind-but-on-time one is "ok", and anything the |
| -// box could not say (no object, an unknown state, an older console) is a dash, |
| -// never "ok". DRAFT copy, pending the owner's copy review. |
| -func replicaCell(r *apiReplica) string { |
| - if r == nil { |
| - return "-" |
| - } |
| - switch r.State { |
| - case "level", "parked", "behind": |
| - return "ok" |
| - case "late": |
| - if r.BehindSecs != nil { |
| - return fmtDur(time.Duration(*r.BehindSecs)*time.Second) + " behind" |
| - } |
| - return "late" |
| - } |
| - return "-" |
| -} |
| - |
| const ( |
| - instancesListHeader = "HOST\tREGION\tCHANNEL\tPOOL\tINSTALLED\tSERVING\tHEALTH\tUPSTREAMS\tDB\tREPLICA\tCREATED\tBY\tREASON" |
| + instancesListHeader = "HOST\tREGION\tCHANNEL\tPOOL\tINSTALLED\tSERVING\tHEALTH\tUPSTREAMS\tDB\tCREATED\tBY\tREASON" |
| instancesListRetryHint = "a build that failed is tried again on every box with carlos promote --app %s --retry <version> <channel>, or on one instance with carlos instances retry --app %s <host>" |
| ) |
| @@ -579,8 +558,8 @@ func runInstancesList(args []string) { |
| if rowHealth == "" { |
| rowHealth = "—" // an old console: the field predates it, not "unhealthy" |
| } |
| - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", r.Host, r.Region, r.Channel, r.Pool, |
| - installedCell(r), servingCell(r), rowHealth, upstreamsCell(r.Upstreams), humanBytes(r.DBBytes), replicaCell(r.Replica), |
| + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", r.Host, r.Region, r.Channel, r.Pool, |
| + installedCell(r), servingCell(r), rowHealth, upstreamsCell(r.Upstreams), humanBytes(r.DBBytes), |
| r.CreatedAt, r.CreatedBy, reasonCell(r)) |
| } |
| w.Flush() |
diff --git a/instances_cli_test.go b/instances_cli_test.go| index 772adcc2d..395408395 100644 |
| --- a/instances_cli_test.go |
| +++ b/instances_cli_test.go |
| @@ -9,10 +9,8 @@ package main |
| import ( |
| "context" |
| - "encoding/json" |
| "net/http" |
| "net/http/httptest" |
| - "strings" |
| "testing" |
| "github.com/carlosframework/platform/internal/provision" |
| @@ -99,67 +97,6 @@ func TestUpstreamsCell(t *testing.T) { |
| } |
| } |
| -// TestReplicaCell pins the REPLICA column: "ok" for a replica that is level, |
| -// parked or behind but on time, a duration only for a late one, and a dash |
| -// for everything the box did not say (never "ok"). |
| -func TestReplicaCell(t *testing.T) { |
| - secs := func(n int64) *int64 { return &n } |
| - cases := []struct { |
| - name string |
| - r *apiReplica |
| - want string |
| - }{ |
| - {"absent (an older console, or no box report)", nil, "-"}, |
| - {"unknown", &apiReplica{State: "unknown"}, "-"}, |
| - {"a state from a newer console", &apiReplica{State: "frozen"}, "-"}, |
| - {"level", &apiReplica{State: "level"}, "ok"}, |
| - {"parked", &apiReplica{State: "parked"}, "ok"}, |
| - {"behind", &apiReplica{State: "behind", BehindSecs: secs(120)}, "ok"}, |
| - {"late", &apiReplica{State: "late", BehindSecs: secs(2*3600 + 10*60)}, "2h10m behind"}, |
| - {"late by minutes", &apiReplica{State: "late", BehindSecs: secs(17*60 + 5)}, "17m05s behind"}, |
| - {"late without a figure", &apiReplica{State: "late"}, "late"}, |
| - } |
| - for _, c := range cases { |
| - if got := replicaCell(c.r); got != c.want { |
| - t.Errorf("replicaCell(%s) = %q, want %q", c.name, got, c.want) |
| - } |
| - } |
| -} |
| - |
| -// TestInstanceListDecodesReplica: the list's "replica" object decodes into |
| -// apiInstanceRecord.Replica, a record without the key decodes to nil (the |
| -// dash above), and the header puts REPLICA right after DB. |
| -func TestInstanceListDecodesReplica(t *testing.T) { |
| - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| - json.NewEncoder(w).Encode(map[string]any{ |
| - "instances": []map[string]any{ |
| - {"host": "late.example.com", "replica": map[string]any{"state": "late", "behind_secs": 7800, "interval_secs": 60}}, |
| - {"host": "old.example.com"}, |
| - }, |
| - "truncated": false, "total": 2, |
| - }) |
| - })) |
| - defer ts.Close() |
| - |
| - s := newTestSession(ts.URL) |
| - recs, _, _, err := s.instanceList(context.Background(), apiAcct{Seg: "bah"}, "app", "", "") |
| - if err != nil { |
| - t.Fatalf("instanceList: %v", err) |
| - } |
| - if len(recs) != 2 { |
| - t.Fatalf("recs = %+v, want 2", recs) |
| - } |
| - if got := replicaCell(recs[0].Replica); got != "2h10m behind" { |
| - t.Errorf("late record cell = %q, want %q", got, "2h10m behind") |
| - } |
| - if recs[1].Replica != nil || replicaCell(recs[1].Replica) != "-" { |
| - t.Errorf("record without a replica key = %+v, want nil and a dash", recs[1].Replica) |
| - } |
| - if !strings.Contains(instancesListHeader, "\tDB\tREPLICA\t") { |
| - t.Errorf("header = %q, want REPLICA immediately after DB", instancesListHeader) |
| - } |
| -} |
| - |
| // TestParseAddrList pins the given-vs-absent split -addr/-addrs share |
| // (Task 8): a flag never named on the command line must return nil (the |
| // wire's "leave/default"), never mistaken for the empty list an explicitly |
diff --git a/instances_e2e_test.go b/instances_e2e_test.go| index 0f61b40ac..8a7720e5d 100644 |
| --- a/instances_e2e_test.go |
| +++ b/instances_e2e_test.go |
| @@ -203,9 +203,6 @@ func TestE2EInstancesThroughConsole(t *testing.T) { |
| if !strings.Contains(listOut, "UPSTREAMS") { |
| t.Fatalf("list stdout = %q, want an UPSTREAMS column header", listOut) |
| } |
| - if !strings.Contains(listOut, "REPLICA") { |
| - t.Fatalf("list stdout = %q, want a REPLICA column header", listOut) |
| - } |
| if !strings.Contains(listOut, haHost) { |
| t.Fatalf("list stdout = %q, want the multi-upstream host listed", listOut) |
| } |
diff --git a/internal/activator/replicalevel.go b/internal/activator/replicalevel.go| deleted file mode 100644 |
| index 074935aa5..000000000 |
| --- a/internal/activator/replicalevel.go |
| +++ /dev/null |
| @@ -1,70 +0,0 @@ |
| -package activator |
| - |
| -import "time" |
| - |
| -// ReplicaLevel is what the activator can say about one hibernating host's |
| -// replica without asking the bucket. |
| -type ReplicaLevel struct { |
| - // LevelAt is the last time the host's running replicator had nothing left |
| - // to ship (lease.Replicator.Level). Zero when it has not managed that |
| - // since the wake, and for a parked host. |
| - LevelAt time.Time |
| - // Empty says the database holds nothing a replicator could ship yet, so |
| - // a zero LevelAt is not a replica falling behind. |
| - Empty bool |
| - // Interval is the host's replication-class sync interval; 0 is stream. |
| - // A replica is not late until it trails by more than this. |
| - Interval time.Duration |
| - // Parked says the host is asleep and its last stop ended with a final |
| - // sync that succeeded: nothing can be waiting to ship. |
| - Parked bool |
| -} |
| - |
| -// leveller is the part of a running replicator ReplicaLevels reads. Only the |
| -// activator's own embedded replicator has it: a unit-backed host's Rep does |
| -// not, because the box-wide replicator owns that database. |
| -type leveller interface { |
| - Level() (at time.Time, empty bool) |
| -} |
| - |
| -// ReplicaLevels reports every host this activator replicates itself: awake |
| -// hosts whose replicator can state its level, and sleeping hosts whose last |
| -// stop is remembered as clean. A unit-backed host is absent, and so is a |
| -// sleeping host whose stop this process did not see or could not prove: |
| -// absent is "cannot say". |
| -func (a *Service) ReplicaLevels() map[string]ReplicaLevel { |
| - type awake struct { |
| - rep leveller |
| - interval time.Duration |
| - } |
| - out := map[string]ReplicaLevel{} |
| - running := map[string]awake{} |
| - |
| - a.mu.Lock() |
| - for host, o := range a.stopHistory { |
| - if o != nil && o.replicaClosed && !o.uncertain { |
| - out[host] = ReplicaLevel{Parked: true} |
| - } |
| - } |
| - for host, e := range a.entries { |
| - // Any live entry replaces what the last stop said: the host is |
| - // waking, awake or stopping, and its replica can be behind again. |
| - delete(out, host) |
| - if e.state != actAwake || e.rep == nil { |
| - continue |
| - } |
| - if rep, ok := e.rep.(leveller); ok { |
| - running[host] = awake{rep, replicationSyncInterval(e.route.Replication)} |
| - } |
| - } |
| - a.mu.Unlock() |
| - |
| - // Asked outside the lock: a replicator's position can cost a directory |
| - // read, and nothing here needs the entry to still be awake by the time |
| - // it answers. |
| - for host, r := range running { |
| - at, empty := r.rep.Level() |
| - out[host] = ReplicaLevel{LevelAt: at, Empty: empty, Interval: r.interval} |
| - } |
| - return out |
| -} |
diff --git a/internal/activator/replicalevel_test.go b/internal/activator/replicalevel_test.go| deleted file mode 100644 |
| index 7841e0b1e..000000000 |
| --- a/internal/activator/replicalevel_test.go |
| +++ /dev/null |
| @@ -1,60 +0,0 @@ |
| -package activator |
| - |
| -import ( |
| - "context" |
| - "testing" |
| - "time" |
| - |
| - "github.com/carlosframework/platform/internal/registry" |
| -) |
| - |
| -type levelRep struct { |
| - at time.Time |
| - empty bool |
| -} |
| - |
| -func (levelRep) Err() error { return nil } |
| -func (levelRep) Close(context.Context) error { return nil } |
| -func (r levelRep) Level() (time.Time, bool) { return r.at, r.empty } |
| - |
| -type silentRep struct{} |
| - |
| -func (silentRep) Err() error { return nil } |
| -func (silentRep) Close(context.Context) error { return nil } |
| - |
| -func TestReplicaLevels(t *testing.T) { |
| - at := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) |
| - clean := &stopObservation{mainStop: true, mainExited: true, sidecarStop: true, sidecarExited: true, replicaClosed: true} |
| - a := &Service{ |
| - entries: map[string]*actEntry{ |
| - "awake.example.com": {host: "awake.example.com", state: actAwake, rep: levelRep{at: at}, route: registry.Route{Replication: replicationBatched}}, |
| - "fresh.example.com": {host: "fresh.example.com", state: actAwake, rep: levelRep{}}, |
| - "empty.example.com": {host: "empty.example.com", state: actAwake, rep: levelRep{empty: true}}, |
| - "unit.example.com": {host: "unit.example.com", state: actAwake, rep: silentRep{}}, |
| - "waking.example.com": {host: "waking.example.com", state: actWaking}, |
| - "stopping.example.com": {host: "stopping.example.com", state: actStopping, rep: levelRep{at: at}}, |
| - "rewoken.example.com": {host: "rewoken.example.com", state: actWaking}, |
| - }, |
| - stopHistory: map[string]*stopObservation{ |
| - "parked.example.com": clean, |
| - "rewoken.example.com": clean, |
| - "unproven.example.com": {mainStop: true, mainExited: true, sidecarStop: true, sidecarExited: true, replicaClosed: true, uncertain: true}, |
| - "syncfailed.example.com": {mainStop: true, mainExited: true, sidecarStop: true, sidecarExited: true}, |
| - }, |
| - } |
| - got := a.ReplicaLevels() |
| - want := map[string]ReplicaLevel{ |
| - "awake.example.com": {LevelAt: at, Interval: 5 * time.Minute}, |
| - "fresh.example.com": {}, |
| - "empty.example.com": {Empty: true}, |
| - "parked.example.com": {Parked: true}, |
| - } |
| - if len(got) != len(want) { |
| - t.Fatalf("ReplicaLevels = %+v, want exactly %+v", got, want) |
| - } |
| - for host, w := range want { |
| - if g, ok := got[host]; !ok || g != w { |
| - t.Errorf("%s = %+v (present %v), want %+v", host, g, ok, w) |
| - } |
| - } |
| -} |
diff --git a/internal/console/alertdetail_test.go b/internal/console/alertdetail_test.go| new file mode 100644 |
| index 000000000..2af025d52 |
| --- /dev/null |
| +++ b/internal/console/alertdetail_test.go |
| @@ -0,0 +1,76 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "strings" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/metrics" |
| +) |
| + |
| +// TestAlertTextForRestarts: the restarts rule says what its value means, and |
| +// an unknown rule still prints a bare number. |
| +func TestAlertTextForRestarts(t *testing.T) { |
| + for _, c := range []struct { |
| + rule string |
| + value float64 |
| + want string |
| + }{ |
| + {"restarts", 5, "5 restarts in 10 minutes"}, |
| + {"somefuturerule", 3, "3"}, |
| + } { |
| + if got := alertValueText(c.rule, c.value); got != c.want { |
| + t.Errorf("alertValueText(%q, %v) = %q, want %q", c.rule, c.value, got, c.want) |
| + } |
| + } |
| +} |
| + |
| +// TestAlertMessageDetailLine: a firing alert that carries a detail names it |
| +// right after the value, worded for its rule; a clear (the box zeroes the |
| +// state) and a rule with no detail add no line. |
| +func TestAlertMessageDetailLine(t *testing.T) { |
| + at := time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC) |
| + firing := func(detail string) metrics.RuleState { |
| + return metrics.RuleState{Firing: true, Value: 3, FiredAt: at, Detail: detail} |
| + } |
| + for _, c := range []struct { |
| + rule, detail, want string |
| + }{ |
| + {"restarts", "carlos-litestream.service", "Value at the transition: 3 restarts in 10 minutes\nUnit: carlos-litestream.service\nTime (UTC)"}, |
| + {"somefuturerule", "thing", "Value at the transition: 3\nDetail: thing\nTime (UTC)"}, |
| + {"restarts", "", "Value at the transition: 3 restarts in 10 minutes\nTime (UTC)"}, |
| + } { |
| + _, body := alertMessage("box-a", c.rule, firing(c.detail), at) |
| + if !strings.Contains(body, c.want) { |
| + t.Errorf("%s/%q body missing %q:\n%s", c.rule, c.detail, c.want, body) |
| + } |
| + } |
| + _, body := alertMessage("box-a", "restarts", metrics.RuleState{}, at) |
| + if strings.Contains(body, "Unit:") { |
| + t.Errorf("a clear invented a detail line:\n%s", body) |
| + } |
| +} |
| + |
| +// TestSweepAlertsCarriesDetail: the dashboard's row keeps the rule's detail. |
| +func TestSweepAlertsCarriesDetail(t *testing.T) { |
| + af := newAlertFixture(t) |
| + af.writeAlert(t, metrics.AlertState{ |
| + V: 1, Box: "box-a", UpdatedAt: time.Now(), |
| + Rules: map[string]metrics.RuleState{ |
| + "restarts": {Firing: true, Value: 4, FiredAt: time.Now(), Detail: "carlos-litestream.service"}, |
| + "disk": {Firing: true, Value: 0.9, FiredAt: time.Now()}, |
| + }, |
| + }) |
| + rows, err := af.srv.sweepAlerts(context.Background()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + got := map[string]string{} |
| + for _, r := range rows { |
| + got[r.Rule] = r.Detail |
| + } |
| + if len(rows) != 2 || got["restarts"] != "carlos-litestream.service" || got["disk"] != "" { |
| + t.Fatalf("rows = %+v", rows) |
| + } |
| +} |
diff --git a/internal/console/alertwatch.go b/internal/console/alertwatch.go| index 5d1c0118f..025702121 100644 |
| --- a/internal/console/alertwatch.go |
| +++ b/internal/console/alertwatch.go |
| @@ -310,8 +310,6 @@ func alertValueText(rule string, value float64) string { |
| return geoipReasonText(int(value)) |
| case "restarts": |
| return fmt.Sprintf("%d restarts in 10 minutes", int(value)) |
| - case "replica": |
| - return humanAge(time.Duration(value)*time.Second) + " overdue" |
| default: |
| return strconv.FormatFloat(value, 'g', -1, 64) |
| } |
| @@ -348,8 +346,6 @@ func alertDetailLine(rule, detail string) string { |
| return "" |
| case rule == "restarts": |
| return "Unit: " + detail + "\n" |
| - case rule == "replica": |
| - return "Database: " + detail + "\n" |
| } |
| return "Detail: " + detail + "\n" |
| } |
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go| index c2c453c02..3c4b3eb0c 100644 |
| --- a/internal/console/apiinstances.go |
| +++ b/internal/console/apiinstances.go |
| @@ -195,12 +195,6 @@ type instanceListRecord struct { |
| // box has sized it: not minted yet, no database file yet, or a route |
| // whose database lives somewhere the platform does not know about. |
| DBBytes *int64 `json:"db_bytes,omitempty"` |
| - // Replica is where this instance's off-box replica stands as its box |
| - // last reported it (status.BoxMetrics.RouteReplicas, discussion #284), |
| - // under the same staleness rule as DBBytes. Absent when no fresh box |
| - // speaks for the host: that is "unknown", and a client must never read it |
| - // as "fine". |
| - Replica *replicaView `json:"replica,omitempty"` |
| // Rollout is this host's rollout as its box last reported it |
| // (zero-downtime spec §9.1), joined under StagedVersion's staleness rule. |
| // Absent when no fresh box reports one: a box from before plan 5, one |
| @@ -355,7 +349,6 @@ type boxJoin struct { |
| serving map[string]string |
| upstreams map[string][]status.UpstreamState |
| dbBytes map[string]int64 |
| - replicas map[string]replicaJoin |
| rollouts map[string]rolloutView |
| updates map[string]updatesJoinView |
| } |
| @@ -470,75 +463,6 @@ func (j boxJoin) dbSize(host string) *int64 { |
| return &n |
| } |
| -// replicaLateAfter is how far past its sync interval a replica may trail |
| -// before it is called late. It matches the box's replica alert rule |
| -// (metrics.DefaultRules().ReplicaSecs = 900) and internal/status's |
| -// replHealthyWithin, so the list, the alert and the status page agree on when |
| -// a replica has fallen behind. |
| -const replicaLateAfter = 15 * time.Minute |
| - |
| -// replicaJoin is one host's replica report with the time of the snapshot it |
| -// came from. |
| -type replicaJoin struct { |
| - r status.RouteReplica |
| - at time.Time |
| -} |
| - |
| -// replicaView is a replica as a member sees it. State is one of "level", |
| -// "behind", "late", "parked" or "unknown"; BehindSecs is set only for "behind" |
| -// and "late". |
| -type replicaView struct { |
| - State string `json:"state"` |
| - LevelAt *time.Time `json:"level_at,omitempty"` |
| - BehindSince *time.Time `json:"behind_since,omitempty"` |
| - BehindSecs *int64 `json:"behind_secs,omitempty"` |
| - IntervalSecs int64 `json:"interval_secs,omitempty"` |
| -} |
| - |
| -// replicaState is the one place a box's replica report becomes a word. now is |
| -// the caller's clock, not read here, so a table of cases can pin it. A |
| -// parked replica is level by construction (its final sync succeeded). A |
| -// replica with BehindSince trails by the time since then. One with only |
| -// LevelAt trails by at most the time since then, so that is the bound the |
| -// word is judged on. A report with neither says nothing yet. A replica is on |
| -// time while it trails by no more than its own interval; replicaLateAfter is |
| -// the slack past that. |
| -func replicaState(r status.RouteReplica, now time.Time) (state string, behind time.Duration) { |
| - interval := time.Duration(r.IntervalSecs) * time.Second |
| - switch { |
| - case r.Parked: |
| - return "parked", 0 |
| - case r.BehindSince != nil: |
| - behind = now.Sub(*r.BehindSince) |
| - if behind-interval > replicaLateAfter { |
| - return "late", behind |
| - } |
| - return "behind", behind |
| - case r.LevelAt != nil: |
| - behind = now.Sub(*r.LevelAt) |
| - if behind-interval > replicaLateAfter { |
| - return "late", behind |
| - } |
| - return "level", behind |
| - } |
| - return "unknown", 0 |
| -} |
| - |
| -// replica is the join's answer for host, nil when no fresh box reported one. |
| -func (j boxJoin) replica(host string, now time.Time) *replicaView { |
| - rj, ok := j.replicas[host] |
| - if !ok { |
| - return nil |
| - } |
| - state, behind := replicaState(rj.r, now) |
| - v := &replicaView{State: state, LevelAt: rj.r.LevelAt, BehindSince: rj.r.BehindSince, IntervalSecs: rj.r.IntervalSecs} |
| - if state == "behind" || state == "late" { |
| - secs := int64(behind / time.Second) |
| - v.BehindSecs = &secs |
| - } |
| - return v |
| -} |
| - |
| // loadBoxJoin builds boxJoin from every NON-STALE box's own |
| // private Metrics block (status/boxes/*.json), through status.LoadBoxes — |
| // the same loader the operator fleet page's HostRows reads through |
| @@ -672,7 +596,7 @@ func recordFleets(recs []provision.Record) map[string]bool { |
| // has its boxes reads them once. |
| func joinBoxes(boxes []status.Box, now time.Time) boxJoin { |
| j := boxJoin{versions: map[string]string{}, serving: map[string]string{}, upstreams: map[string][]status.UpstreamState{}, |
| - dbBytes: map[string]int64{}, replicas: map[string]replicaJoin{}, rollouts: map[string]rolloutView{}} |
| + dbBytes: map[string]int64{}, rollouts: map[string]rolloutView{}} |
| for _, b := range boxes { |
| if b.Metrics == nil || status.BoxStale(b, now, status.SnapshotPeriod) { |
| continue |
| @@ -689,9 +613,6 @@ func joinBoxes(boxes []status.Box, now time.Time) boxJoin { |
| for host, n := range b.Metrics.RouteDBBytes { |
| j.dbBytes[host] = n |
| } |
| - for host, r := range b.Metrics.RouteReplicas { |
| - j.replicas[host] = replicaJoin{r: r, at: b.GeneratedAt} |
| - } |
| j.foldRollout(b, true) |
| j.foldUpdates(b, true) |
| } |
| @@ -845,8 +766,7 @@ func (s *Server) handleAPIInstanceList(w http.ResponseWriter, r *http.Request) { |
| acct.Sqid, app, len(skipped), skipped) |
| relevant = nil |
| } |
| - now := time.Now() |
| - join, joinComplete := s.loadRolloutJoin(ctx, acct.Sqid, now, relevant) |
| + join, joinComplete := s.loadRolloutJoin(ctx, acct.Sqid, time.Now(), relevant) |
| served := loadEdgesJoin(ctx, s.cfg.Releases) |
| // The app's update policy, read once: an absent manifest is no policy, |
| // and a read error is logged and treated the same. The list never fails |
| @@ -887,7 +807,6 @@ func (s *Server) handleAPIInstanceList(w http.ResponseWriter, r *http.Request) { |
| matched = append(matched, instanceListRecord{Record: rec, |
| StagedVersion: join.versions[rec.Host], Health: h, Upstreams: join.upstreams[rec.Host], |
| ServedBy: served[rec.Host], Gated: rec.Gate != "", DBBytes: join.dbSize(rec.Host), |
| - Replica: join.replica(rec.Host, now), |
| Rollout: join.rollout(rec.Host, acct.Sqid, app), |
| UpdateReport: join.update(rec.Host, acct.Sqid, app), |
| SettingsDefault: provision.EffectiveUpdates(rec, policy).Default()}) |
diff --git a/internal/console/overview.go b/internal/console/overview.go| index 9229a8ed4..03db7eba9 100644 |
| --- a/internal/console/overview.go |
| +++ b/internal/console/overview.go |
| @@ -5,9 +5,7 @@ |
| package console |
| import ( |
| - "fmt" |
| "strings" |
| - "time" |
| "github.com/carlosframework/platform/internal/registry" |
| ) |
| @@ -234,38 +232,6 @@ type InstanceRow struct { |
| // plain word — the column's whole job. |
| HealthWord string |
| Restartable bool |
| - |
| - // ReplicaText is the Replica column's cell, "" when the route's box has |
| - // not said where its off-box replica stands. ReplicaLate marks the one |
| - // state worth drawing the eye to. Both come from attachReplicas. |
| - ReplicaText string |
| - ReplicaLate bool |
| -} |
| - |
| -// attachReplicas fills each row's Replica cell from the box join. A host no |
| -// fresh box speaks for keeps an empty cell: unknown is blank, never "ok". |
| -func attachReplicas(rows []InstanceRow, join boxJoin, now time.Time) { |
| - for i := range rows { |
| - rows[i].ReplicaText, rows[i].ReplicaLate = replicaCellText(join.replica(rows[i].Host, now)) |
| - } |
| -} |
| - |
| -// replicaCellText is the Replica column's wording, the only place it lives. |
| -// DRAFT copy, pending the owner's copy review. |
| -func replicaCellText(v *replicaView) (text string, late bool) { |
| - if v == nil { |
| - return "", false |
| - } |
| - switch v.State { |
| - case "level", "parked", "behind": |
| - return "ok", false |
| - case "late": |
| - if v.BehindSecs != nil { |
| - return fmt.Sprintf("%s behind", humanAge(time.Duration(*v.BehindSecs)*time.Second)), true |
| - } |
| - return "late", true |
| - } |
| - return "", false |
| } |
| // assembleInstanceRows joins the app's routes with the sweep's readings, in |
diff --git a/internal/console/replicafresh_test.go b/internal/console/replicafresh_test.go| deleted file mode 100644 |
| index 134c21833..000000000 |
| --- a/internal/console/replicafresh_test.go |
| +++ /dev/null |
| @@ -1,261 +0,0 @@ |
| -package console |
| - |
| -import ( |
| - "context" |
| - "encoding/json" |
| - "net/http" |
| - "strings" |
| - "testing" |
| - "time" |
| - |
| - "github.com/carlosframework/platform/internal/metrics" |
| - "github.com/carlosframework/platform/internal/registry" |
| - "github.com/carlosframework/platform/internal/status" |
| -) |
| - |
| -// TestReplicaState pins every branch of the one function that turns a box's |
| -// replica report into a word, including the edges: exactly 15 minutes past |
| -// the interval is not yet late, and the interval itself is respected. |
| -func TestReplicaState(t *testing.T) { |
| - now := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) |
| - ago := func(d time.Duration) *time.Time { v := now.Add(-d); return &v } |
| - cases := []struct { |
| - name string |
| - r status.RouteReplica |
| - want string |
| - wantBehind time.Duration |
| - }{ |
| - {"parked", status.RouteReplica{Parked: true}, "parked", 0}, |
| - {"parked wins over a stale level", status.RouteReplica{Parked: true, LevelAt: ago(48 * time.Hour)}, "parked", 0}, |
| - {"parked wins over behind", status.RouteReplica{Parked: true, BehindSince: ago(48 * time.Hour)}, "parked", 0}, |
| - {"behind, recent", status.RouteReplica{BehindSince: ago(2 * time.Minute)}, "behind", 2 * time.Minute}, |
| - {"behind, exactly 15m past a zero interval", status.RouteReplica{BehindSince: ago(15 * time.Minute)}, "behind", 15 * time.Minute}, |
| - {"behind, one second over 15m", status.RouteReplica{BehindSince: ago(15*time.Minute + time.Second)}, "late", 15*time.Minute + time.Second}, |
| - {"behind, long interval 23h", status.RouteReplica{BehindSince: ago(23 * time.Hour), IntervalSecs: 86400}, "behind", 23 * time.Hour}, |
| - {"behind, long interval exactly 24h15m", status.RouteReplica{BehindSince: ago(24*time.Hour + 15*time.Minute), IntervalSecs: 86400}, "behind", 24*time.Hour + 15*time.Minute}, |
| - {"behind, long interval 24h16m", status.RouteReplica{BehindSince: ago(24*time.Hour + 16*time.Minute), IntervalSecs: 86400}, "late", 24*time.Hour + 16*time.Minute}, |
| - {"behind wins over level", status.RouteReplica{BehindSince: ago(time.Minute), LevelAt: ago(time.Hour)}, "behind", time.Minute}, |
| - {"level, recent", status.RouteReplica{LevelAt: ago(time.Minute)}, "level", time.Minute}, |
| - {"level, exactly 15m", status.RouteReplica{LevelAt: ago(15 * time.Minute)}, "level", 15 * time.Minute}, |
| - {"level, one second over 15m", status.RouteReplica{LevelAt: ago(15*time.Minute + time.Second)}, "late", 15*time.Minute + time.Second}, |
| - {"level, long interval 23h", status.RouteReplica{LevelAt: ago(23 * time.Hour), IntervalSecs: 86400}, "level", 23 * time.Hour}, |
| - {"level, long interval 24h16m", status.RouteReplica{LevelAt: ago(24*time.Hour + 16*time.Minute), IntervalSecs: 86400}, "late", 24*time.Hour + 16*time.Minute}, |
| - {"nothing known", status.RouteReplica{}, "unknown", 0}, |
| - {"interval alone says nothing", status.RouteReplica{IntervalSecs: 60}, "unknown", 0}, |
| - } |
| - for _, c := range cases { |
| - got, behind := replicaState(c.r, now) |
| - if got != c.want || behind != c.wantBehind { |
| - t.Errorf("%s: replicaState = %q, %v; want %q, %v", c.name, got, behind, c.want, c.wantBehind) |
| - } |
| - } |
| -} |
| - |
| -// TestReplicaCellText pins the table's wording: unknown is blank, never "ok". |
| -func TestReplicaCellText(t *testing.T) { |
| - secs := func(n int64) *int64 { return &n } |
| - cases := []struct { |
| - name string |
| - v *replicaView |
| - want string |
| - wantLate bool |
| - }{ |
| - {"no report", nil, "", false}, |
| - {"unknown", &replicaView{State: "unknown"}, "", false}, |
| - {"level", &replicaView{State: "level"}, "ok", false}, |
| - {"parked", &replicaView{State: "parked"}, "ok", false}, |
| - {"behind", &replicaView{State: "behind", BehindSecs: secs(300)}, "ok", false}, |
| - {"late", &replicaView{State: "late", BehindSecs: secs(2*3600 + 10*60)}, "2h10m behind", true}, |
| - {"late without a figure", &replicaView{State: "late"}, "late", true}, |
| - {"a state from the future", &replicaView{State: "frozen"}, "", false}, |
| - } |
| - for _, c := range cases { |
| - got, late := replicaCellText(c.v) |
| - if got != c.want || late != c.wantLate { |
| - t.Errorf("%s: replicaCellText = %q, %v; want %q, %v", c.name, got, late, c.want, c.wantLate) |
| - } |
| - } |
| -} |
| - |
| -// TestAPIInstanceListReplica: replica joins from a FRESH box's |
| -// Metrics.RouteReplicas, is absent for a host the box did not speak for, and |
| -// is dropped for a host whose only reporting box has gone stale, the same |
| -// rules as db_bytes. |
| -func TestAPIInstanceListReplica(t *testing.T) { |
| - fx, token := instanceFixture(t, "code-inst-replica-state") |
| - enableInstances(t, fx, "titogo", titoManifest()) |
| - u := instancesURL(fx, "bab", "titogo") |
| - for _, h := range []string{"shop.go.tito.io", "cafe.go.tito.io", "late.go.tito.io", "stale.go.tito.io"} { |
| - if code, _, out := rawReq(t, http.MethodPost, u, token, |
| - `{"host":"`+h+`","region":"ie"}`); code != http.StatusCreated { |
| - t.Fatalf("create %s = %d %s", h, code, out) |
| - } |
| - } |
| - now := time.Now() |
| - fresh := status.CollectBox(nil, status.Probes{MachineID: "box1"}, now) |
| - levelAt := now.Add(-time.Minute).UTC().Truncate(time.Second) |
| - behindSince := now.Add(-3 * time.Hour).UTC().Truncate(time.Second) |
| - fresh.Metrics = &status.BoxMetrics{RouteReplicas: map[string]status.RouteReplica{ |
| - "shop.go.tito.io": {LevelAt: &levelAt, IntervalSecs: 60}, |
| - "late.go.tito.io": {BehindSince: &behindSince}, |
| - }} |
| - if err := status.WriteBox(context.Background(), fx.rel, fresh); err != nil { |
| - t.Fatal(err) |
| - } |
| - stale := status.CollectBox(nil, status.Probes{MachineID: "box2"}, now.Add(-time.Hour)) |
| - stale.Metrics = &status.BoxMetrics{RouteReplicas: map[string]status.RouteReplica{ |
| - "stale.go.tito.io": {LevelAt: &levelAt}, |
| - }} |
| - if err := status.WriteBox(context.Background(), fx.rel, stale); err != nil { |
| - t.Fatal(err) |
| - } |
| - |
| - code, _, out := rawReq(t, http.MethodGet, u, token, "") |
| - if code != http.StatusOK { |
| - t.Fatalf("list = %d %s", code, out) |
| - } |
| - var list struct { |
| - Instances []map[string]any `json:"instances"` |
| - } |
| - if err := json.Unmarshal([]byte(out), &list); err != nil { |
| - t.Fatalf("list body %s: %v", out, err) |
| - } |
| - got := map[string]map[string]any{} |
| - for _, r := range list.Instances { |
| - got[r["host"].(string)] = r |
| - } |
| - shop, _ := got["shop.go.tito.io"]["replica"].(map[string]any) |
| - if shop == nil || shop["state"] != "level" || shop["interval_secs"] != float64(60) || shop["level_at"] != levelAt.Format(time.RFC3339) { |
| - t.Errorf("shop replica = %v, want a level object with its level_at and interval", got["shop.go.tito.io"]["replica"]) |
| - } |
| - if _, has := shop["behind_secs"]; has { |
| - t.Errorf("a level replica carries behind_secs: %v", shop) |
| - } |
| - late, _ := got["late.go.tito.io"]["replica"].(map[string]any) |
| - if late == nil || late["state"] != "late" || late["behind_since"] != behindSince.Format(time.RFC3339) { |
| - t.Fatalf("late replica = %v, want a late object", got["late.go.tito.io"]["replica"]) |
| - } |
| - if secs, _ := late["behind_secs"].(float64); secs < 3*3600 || secs > 3*3600+60 { |
| - t.Errorf("late behind_secs = %v, want about 3 hours", late["behind_secs"]) |
| - } |
| - for _, h := range []string{"cafe.go.tito.io", "stale.go.tito.io"} { |
| - if v, ok := got[h]["replica"]; ok { |
| - t.Errorf("%s replica = %v, want the key absent", h, v) |
| - } |
| - } |
| -} |
| - |
| -// TestAppInstancesTableReplicaColumn: the Instances tab shows the Replica |
| -// column from the same join, "ok" for a replica that is level, a flagged |
| -// duration for a late one, and nothing for a host no box speaks for. |
| -func TestAppInstancesTableReplicaColumn(t *testing.T) { |
| - fx := newConsoleFixture(t) |
| - for _, h := range []string{"level.kass.test", "late.kass.test", "mute.kass.test"} { |
| - fx.addRoute(t, registry.Route{Host: h, App: "kass", Account: "bab", Kind: "instance", Status: "ready"}) |
| - } |
| - now := time.Now() |
| - levelAt := now.Add(-time.Minute) |
| - behindSince := now.Add(-2*time.Hour - 10*time.Minute) |
| - box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, now) |
| - box.Metrics = &status.BoxMetrics{RouteReplicas: map[string]status.RouteReplica{ |
| - "level.kass.test": {LevelAt: &levelAt}, |
| - "late.kass.test": {BehindSince: &behindSince}, |
| - }} |
| - if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| - t.Fatal(err) |
| - } |
| - c := signInPaul(t, fx) |
| - st, _, body := get(t, c, fx.u("/apps/kass/instances")) |
| - if st != http.StatusOK { |
| - t.Fatalf("GET instances = %d:\n%s", st, body) |
| - } |
| - if !strings.Contains(body, `<th scope="col">Replica</th>`) { |
| - t.Fatalf("no Replica column header:\n%s", body) |
| - } |
| - row := func(host string) string { |
| - i := strings.Index(body, ">"+host+"</a>") |
| - if i < 0 { |
| - t.Fatalf("no row for %s:\n%s", host, body) |
| - } |
| - j := strings.Index(body[i:], "</tr>") |
| - return body[i : i+j] |
| - } |
| - if r := row("level.kass.test"); !strings.HasSuffix(strings.TrimSpace(r), "<td>ok</td>") { |
| - t.Errorf("level row has no ok cell:\n%s", r) |
| - } |
| - if r := row("late.kass.test"); !strings.Contains(r, `chip-health-attention">2h10m behind</span>`) { |
| - t.Errorf("late row has no flagged duration:\n%s", r) |
| - } |
| - if r := row("mute.kass.test"); !strings.HasSuffix(strings.TrimSpace(r), "<td></td>") { |
| - t.Errorf("a host no box speaks for must have an empty replica cell:\n%s", r) |
| - } |
| -} |
| - |
| -// TestAlertTextForRestartsAndReplica: the two newer rules say what their |
| -// value means, and an unknown rule still prints a bare number. |
| -func TestAlertTextForRestartsAndReplica(t *testing.T) { |
| - for _, c := range []struct { |
| - rule string |
| - value float64 |
| - want string |
| - }{ |
| - {"restarts", 5, "5 restarts in 10 minutes"}, |
| - {"replica", 4320, "1h12m overdue"}, |
| - {"replica", 905, "15m overdue"}, |
| - {"somefuturerule", 3, "3"}, |
| - } { |
| - if got := alertValueText(c.rule, c.value); got != c.want { |
| - t.Errorf("alertValueText(%q, %v) = %q, want %q", c.rule, c.value, got, c.want) |
| - } |
| - } |
| -} |
| - |
| -// TestAlertMessageDetailLine: a firing alert that carries a detail names it |
| -// right after the value, worded for its rule; a clear (the box zeroes the |
| -// state) and a rule with no detail add no line. |
| -func TestAlertMessageDetailLine(t *testing.T) { |
| - at := time.Date(2026, 10, 13, 9, 0, 0, 0, time.UTC) |
| - firing := func(detail string) metrics.RuleState { |
| - return metrics.RuleState{Firing: true, Value: 3, FiredAt: at, Detail: detail} |
| - } |
| - for _, c := range []struct { |
| - rule, detail, want string |
| - }{ |
| - {"restarts", "carlos-litestream.service", "Value at the transition: 3 restarts in 10 minutes\nUnit: carlos-litestream.service\nTime (UTC)"}, |
| - {"replica", "/data/shop/app.db", "Database: /data/shop/app.db\nTime (UTC)"}, |
| - {"somefuturerule", "thing", "Value at the transition: 3\nDetail: thing\nTime (UTC)"}, |
| - {"restarts", "", "Value at the transition: 3 restarts in 10 minutes\nTime (UTC)"}, |
| - } { |
| - _, body := alertMessage("box-a", c.rule, firing(c.detail), at) |
| - if !strings.Contains(body, c.want) { |
| - t.Errorf("%s/%q body missing %q:\n%s", c.rule, c.detail, c.want, body) |
| - } |
| - } |
| - _, body := alertMessage("box-a", "restarts", metrics.RuleState{}, at) |
| - if strings.Contains(body, "Unit:") { |
| - t.Errorf("a clear invented a detail line:\n%s", body) |
| - } |
| -} |
| - |
| -// TestSweepAlertsCarriesDetail: the dashboard's row keeps the rule's detail. |
| -func TestSweepAlertsCarriesDetail(t *testing.T) { |
| - af := newAlertFixture(t) |
| - af.writeAlert(t, metrics.AlertState{ |
| - V: 1, Box: "box-a", UpdatedAt: time.Now(), |
| - Rules: map[string]metrics.RuleState{ |
| - "restarts": {Firing: true, Value: 4, FiredAt: time.Now(), Detail: "carlos-litestream.service"}, |
| - "disk": {Firing: true, Value: 0.9, FiredAt: time.Now()}, |
| - }, |
| - }) |
| - rows, err := af.srv.sweepAlerts(context.Background()) |
| - if err != nil { |
| - t.Fatal(err) |
| - } |
| - got := map[string]string{} |
| - for _, r := range rows { |
| - got[r.Rule] = r.Detail |
| - } |
| - if len(rows) != 2 || got["restarts"] != "carlos-litestream.service" || got["disk"] != "" { |
| - t.Fatalf("rows = %+v", rows) |
| - } |
| -} |
diff --git a/internal/console/server.go b/internal/console/server.go| index 15e96185c..d0b388eae 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -3328,15 +3328,6 @@ func (s *Server) handleAppInstances(w http.ResponseWriter, r *http.Request) { |
| p.AppMetrics = s.appResourceMetrics(r.Context(), p.Detail, acct) |
| q := parseInstanceQuery(r.URL.Query()) |
| rows := assembleInstanceRows(p.Detail, acct, p.AppMetrics) |
| - // The Replica column is best-effort like the Database tile: a slow or |
| - // failed status read leaves the cells blank (loadBoxJoin logs it), never |
| - // the page failing. |
| - if len(rows) > 0 { |
| - joinCtx, cancelJoin := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| - now := time.Now() |
| - attachReplicas(rows, loadBoxJoin(joinCtx, s.cfg.Releases, now), now) |
| - cancelJoin() |
| - } |
| for _, row := range rows { |
| if row.Restartable { |
| p.CanRestartApp = true |
diff --git a/internal/console/templates/app_instances.html b/internal/console/templates/app_instances.html| index b96a2f3dd..b3904fcbb 100644 |
| --- a/internal/console/templates/app_instances.html |
| +++ b/internal/console/templates/app_instances.html |
| @@ -31,11 +31,11 @@ |
| {{if .InstanceRows}} |
| <div class="tablewrap"> |
| <table class="instances"> |
| - <thead><tr><th scope="col">Address</th><th scope="col">Health</th><th scope="col">Version</th><th scope="col">Channel</th><th scope="col">CPU</th><th scope="col">Memory</th><th scope="col">Replica</th></tr></thead> |
| + <thead><tr><th scope="col">Address</th><th scope="col">Health</th><th scope="col">Version</th><th scope="col">Channel</th><th scope="col">CPU</th><th scope="col">Memory</th></tr></thead> |
| {{if .InstanceQuery.Group}} |
| {{range .InstanceGroups}} |
| <tbody> |
| - <tr class="instance-group"><th colspan="7" scope="rowgroup"><code>{{.Channel}}</code> <span class="small">{{$.InstanceCopy.Target}}: {{if .TargetErr}}—{{else if .TargetVersion}}<code>{{.TargetVersion}}</code>{{else}}—{{end}} · Instances: {{len .Rows}} / {{.Total}}</span></th></tr> |
| + <tr class="instance-group"><th colspan="6" scope="rowgroup"><code>{{.Channel}}</code> <span class="small">{{$.InstanceCopy.Target}}: {{if .TargetErr}}—{{else if .TargetVersion}}<code>{{.TargetVersion}}</code>{{else}}—{{end}} · Instances: {{len .Rows}} / {{.Total}}</span></th></tr> |
| {{range .Rows}}{{template "instance_row" (instanceRowPage $ .)}}{{end}} |
| </tbody> |
| {{end}} |
| @@ -109,6 +109,5 @@ hibernated instance state stays in the old placement's bucket.</p> |
| <td><span class="cell-value">{{pct .Row.Metrics.CPUFrac}}</span>{{if .Row.Metrics.SparkCPU}}<svg class="cellspark" aria-hidden="true" viewBox="0 0 240 40" preserveAspectRatio="none"><polyline fill="none" stroke="currentColor" points="{{.Row.Metrics.SparkCPU}}"/></svg>{{end}}</td> |
| <td><span class="cell-value">{{bytesOrDash .Row.Metrics.MemBytes}}</span>{{if .Row.Metrics.SparkMem}}<svg class="cellspark" aria-hidden="true" viewBox="0 0 240 40" preserveAspectRatio="none"><polyline fill="none" stroke="currentColor" points="{{.Row.Metrics.SparkMem}}"/></svg>{{end}}</td> |
| {{else}}<td colspan="2"><span class="small">no data</span></td>{{end}} |
| - <td>{{if .Row.ReplicaLate}}<span class="chip chip-health-attention">{{.Row.ReplicaText}}</span>{{else}}{{.Row.ReplicaText}}{{end}}</td> |
| </tr> |
| {{end}} |
diff --git a/internal/lease/lease.go b/internal/lease/lease.go| index dea5e3f7c..61788e219 100644 |
| --- a/internal/lease/lease.go |
| +++ b/internal/lease/lease.go |
| @@ -487,25 +487,3 @@ func Restore(ctx context.Context, dbPath, bucket, prefix, region string) (bool, |
| } |
| return true, nil |
| } |
| - |
| -// Level reports how current the replica is without asking the bucket. |
| -// |
| -// at is the last time a replica sync ended with nothing left to ship: every |
| -// change the database held when that sync began was in the replica when it |
| -// returned. litestream records it at the end of each sync that returns nil |
| -// (an idle database's ticks included, which make no S3 call), so the time |
| -// since it is an upper bound on how far the replica trails. It is zero until |
| -// the first such sync after this replicator started. |
| -// |
| -// empty says the database has no position yet: nothing has been written that |
| -// litestream could ship, and it records no sync for such a database however |
| -// long it runs. A zero at with empty false is a replicator that has data and |
| -// has not yet managed to ship it. |
| -func (r *Replicator) Level() (at time.Time, empty bool) { |
| - at = r.db.LastSuccessfulSyncAt() |
| - if !at.IsZero() { |
| - return at, false |
| - } |
| - pos, err := r.db.Pos() |
| - return at, err == nil && pos.IsZero() |
| -} |
diff --git a/internal/metrics/alert.go b/internal/metrics/alert.go| index fd37d28d5..489952dc4 100644 |
| --- a/internal/metrics/alert.go |
| +++ b/internal/metrics/alert.go |
| @@ -23,16 +23,8 @@ type Rules struct { |
| // observation's window before the restarts rule fires. It is a count, not |
| // a fraction, and the rule clears only on a reading of 0. |
| RestartCount float64 // fire >, default 3 |
| - // ReplicaSecs is how long a database's replica may be overdue (behind, |
| - // beyond its own sync interval) before the replica rule fires. Seconds, |
| - // like LagSecs; it clears replicaClearBy below. |
| - ReplicaSecs int64 // fire >, default 900, immediate |
| } |
| -// replicaClearBy is the replica rule's hysteresis: fired above 15 minutes |
| -// overdue, it clears below 10. |
| -const replicaClearBy = 300 |
| - |
| // DefaultRules are the thresholds shipped with the platform. |
| func DefaultRules() Rules { |
| return Rules{ |
| @@ -46,7 +38,6 @@ func DefaultRules() Rules { |
| // GeoIPSustain: geoip spec §10.4. |
| GeoIPSustain: 30 * time.Minute, |
| RestartCount: 3, |
| - ReplicaSecs: 900, |
| } |
| } |
| @@ -106,12 +97,6 @@ type Observation struct { |
| // tell yet. |
| Restarts *float64 |
| RestartsUnit string |
| - // ReplicaOverdueSecs is how far the box's worst replica is overdue: the |
| - // time since it was last level with its database, less that database's |
| - // sync interval. ReplicaOf names the database or host. 0 says every |
| - // replica the reporter can see is on time; nil says it can see none. |
| - ReplicaOverdueSecs *int64 |
| - ReplicaOf string |
| } |
| // RuleState is one rule's sustain/hysteresis bookkeeping. Since and FiredAt |
| @@ -130,8 +115,7 @@ type RuleState struct { |
| FiredAt time.Time `json:"fired_at,omitempty"` // set on fire, cleared on clear |
| Value float64 `json:"value,omitempty"` // reading at last transition |
| // Detail names what the reading was about, for the rules whose value |
| - // alone does not say: the unit for "restarts", the database or host for |
| - // "replica". Empty for the rest. |
| + // alone does not say: the unit for "restarts". Empty for the rest. |
| Detail string `json:"detail,omitempty"` |
| } |
| @@ -142,7 +126,7 @@ type AlertState struct { |
| V int `json:"v"` // 1 |
| Box string `json:"box"` |
| UpdatedAt time.Time `json:"updated_at"` |
| - Rules map[string]RuleState `json:"rules"` // "disk","mem","cpu","lag","geoip","restarts","replica" |
| + Rules map[string]RuleState `json:"rules"` // "disk","mem","cpu","lag","geoip","restarts" |
| } |
| // Event is emitted on a firing or clearing transition — never on every |
| @@ -323,22 +307,5 @@ func Evaluate(r Rules, st AlertState, obs Observation) (AlertState, []Event) { |
| } |
| } |
| - // replica: the reading is already a duration, so like lag it fires at |
| - // once. It names what is behind, and keeps that name until it clears even |
| - // if another replica becomes the worst meanwhile. |
| - if obs.ReplicaOverdueSecs != nil { |
| - val := float64(*obs.ReplicaOverdueSecs) |
| - cur := out.Rules["replica"] |
| - switch { |
| - case cur.Firing && val < float64(r.ReplicaSecs-replicaClearBy): |
| - of := cur.Detail |
| - out.Rules["replica"] = RuleState{Value: val} |
| - events = append(events, Event{Rule: "replica", Firing: false, Value: val, Box: st.Box, Detail: of}) |
| - case !cur.Firing && val > float64(r.ReplicaSecs): |
| - out.Rules["replica"] = RuleState{Firing: true, Since: obs.At, FiredAt: obs.At, Value: val, Detail: obs.ReplicaOf} |
| - events = append(events, Event{Rule: "replica", Firing: true, Value: val, Box: st.Box, Detail: obs.ReplicaOf}) |
| - } |
| - } |
| - |
| return out, events |
| } |
diff --git a/internal/metrics/alert_test.go b/internal/metrics/alert_test.go| index ec24213bf..92e0a6d4c 100644 |
| --- a/internal/metrics/alert_test.go |
| +++ b/internal/metrics/alert_test.go |
| @@ -289,41 +289,3 @@ func TestRestartsRuleFiresAboveTheCountAndClearsOnlyAtZero(t *testing.T) { |
| t.Fatalf("a quiet rule emitted: %v", ev) |
| } |
| } |
| - |
| -// The replica rule fires once a replica is more than 15 minutes overdue, |
| -// names it, and clears below 10 minutes. No reading changes nothing. |
| -func TestReplicaRuleFiresWhenOverdueAndClearsWithHysteresis(t *testing.T) { |
| - r := DefaultRules() |
| - t0 := time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) |
| - secs := func(v int64) *int64 { return &v } |
| - st := AlertState{V: 1, Box: "b"} |
| - |
| - st, ev := Evaluate(r, st, Observation{At: t0, ReplicaOverdueSecs: secs(900), ReplicaOf: "/data/shop.db"}) |
| - if len(ev) != 0 { |
| - t.Fatalf("fired at 15 minutes, not above: %v", ev) |
| - } |
| - st, ev = Evaluate(r, st, Observation{At: t0.Add(time.Minute), ReplicaOverdueSecs: secs(960), ReplicaOf: "/data/shop.db"}) |
| - if len(ev) != 1 || !ev[0].Firing || ev[0].Rule != "replica" || ev[0].Value != 960 || ev[0].Detail != "/data/shop.db" { |
| - t.Fatalf("above 15 minutes: %+v", ev) |
| - } |
| - // Another database is now the worst; the alert keeps the name it fired on. |
| - st, ev = Evaluate(r, st, Observation{At: t0.Add(2 * time.Minute), ReplicaOverdueSecs: secs(5000), ReplicaOf: "/data/other.db"}) |
| - if len(ev) != 0 || st.Rules["replica"].Detail != "/data/shop.db" { |
| - t.Fatalf("a change of worst replica re-fired or renamed: %v %+v", ev, st.Rules["replica"]) |
| - } |
| - st, ev = Evaluate(r, st, Observation{At: t0.Add(3 * time.Minute)}) |
| - if len(ev) != 0 || !st.Rules["replica"].Firing { |
| - t.Fatalf("no reading changed the rule: %v", ev) |
| - } |
| - st, ev = Evaluate(r, st, Observation{At: t0.Add(4 * time.Minute), ReplicaOverdueSecs: secs(700)}) |
| - if len(ev) != 0 || !st.Rules["replica"].Firing { |
| - t.Fatalf("cleared inside the hysteresis band: %v", ev) |
| - } |
| - st, ev = Evaluate(r, st, Observation{At: t0.Add(5 * time.Minute), ReplicaOverdueSecs: secs(0)}) |
| - if len(ev) != 1 || ev[0].Firing || ev[0].Detail != "/data/shop.db" { |
| - t.Fatalf("on time again must clear and name what had fired: %+v", ev) |
| - } |
| - if rs := st.Rules["replica"]; rs.Firing || rs.Detail != "" || !rs.FiredAt.IsZero() { |
| - t.Fatalf("state after clearing: %+v", rs) |
| - } |
| -} |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index 76329d6f6..4dd9ef44a 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -150,12 +150,6 @@ type Probes struct { |
| // out of Metrics.RouteDBBytes rather than published as 0. |
| DBBytes func(dbPath string) *int64 |
| - // Replicas is host -> what this box knows about that route's off-box |
| - // replica, already assembled by the caller from the activator's |
| - // replicators and the box-wide replica check. Carried straight through |
| - // to Metrics.RouteReplicas. A host the box cannot speak for is absent. |
| - Replicas map[string]RouteReplica |
| - |
| // Layout is the box's layout report (isolated-exec design §8.4): which |
| // mode the box last converged to and, per instance-kind route, the word |
| // the layout pass's own Classify gave it. The caller (internal/host's |
| @@ -597,11 +591,6 @@ type BoxMetrics struct { |
| // readers. A host whose file could not be sized is absent, never 0; the |
| // whole map is omitted when no route could be sized. |
| RouteDBBytes map[string]int64 `json:"route_db_bytes,omitempty"` |
| - // RouteReplicas is host -> the state of that route's off-box replica as |
| - // this box sees it (discussion #284). Hostname-keyed and private like |
| - // RouteDBBytes, and absent for a host the box cannot speak for: no entry |
| - // is "unknown", never "fine". |
| - RouteReplicas map[string]RouteReplica `json:"route_replicas,omitempty"` |
| // Layout is Probes.Layout carried straight through — the box's layout |
| // report, hostname-keyed like RouteVersions and RouteDBBytes above, and |
| // private for the identical reason: BuildFleet copies Box's fields into |
| @@ -710,25 +699,6 @@ type RouteCounts struct { |
| Asleep *int `json:"asleep"` // null when the socket table is unreadable |
| } |
| -// RouteReplica is one route's replica as its box sees it. At most one of |
| -// LevelAt, BehindSince and Parked says where the replica stands; none of them |
| -// set means the box replicates the route but cannot yet say (a host woken |
| -// moments ago, before its first sync). |
| -type RouteReplica struct { |
| - // LevelAt is the last time nothing on the box was waiting to ship for |
| - // this route: the replica held every change the database did. |
| - LevelAt *time.Time `json:"level_at,omitempty"` |
| - // BehindSince is set when the replica is known to trail the database, |
| - // and is the earliest time the box held a change the replica did not. |
| - BehindSince *time.Time `json:"behind_since,omitempty"` |
| - // Parked says the instance is asleep and its final sync succeeded, so |
| - // nothing can be waiting. |
| - Parked bool `json:"parked,omitempty"` |
| - // IntervalSecs is the route's sync interval (0 for a continuous one). A |
| - // replica is on time while it trails by no more than this. |
| - IntervalSecs int64 `json:"interval_secs,omitempty"` |
| -} |
| - |
| // Replication summarizes litestream's furthest-behind lag across this box's |
| // routes, banded to a healthy bool rather than the raw seconds — the raw |
| // number is still carried for the console's private view, but the public |
| @@ -1073,7 +1043,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" || |
| p.InstanceID != "" || p.Platform != "" || p.SystemChannel != "" || p.System != nil || |
| len(routeVersions) > 0 || len(routeServing) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 || |
| - len(routeDBBytes) > 0 || len(p.Replicas) > 0 || p.Layout != nil || len(p.ConfigRefusals) > 0 || len(p.ReleaseRefusals) > 0 || len(p.ProofWaits) > 0 || p.Rollout != nil || p.ConfigGuard != "" || |
| + len(routeDBBytes) > 0 || p.Layout != nil || len(p.ConfigRefusals) > 0 || len(p.ReleaseRefusals) > 0 || len(p.ProofWaits) > 0 || p.Rollout != nil || p.ConfigGuard != "" || |
| p.Alerts != nil || p.GeoIP != nil || len(p.BuildStates) > 0 || p.BuildStatesErr != "" || |
| p.UpdateSettings != 0 || len(p.Updates) > 0 { |
| box.Metrics = &BoxMetrics{ |
| @@ -1105,9 +1075,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| // alongside another known field above, and omitempty drops it. |
| Upstreams: p.Upstreams, |
| RouteDBBytes: routeDBBytes, |
| - // Built fresh by the caller every tick, like Upstreams. |
| - RouteReplicas: p.Replicas, |
| - Layout: p.Layout, |
| + Layout: p.Layout, |
| // A fresh copy: the guard's own slice is its to mutate. |
| ConfigRefusals: append([]ConfigRefusal(nil), p.ConfigRefusals...), |
| ConfigRefusalsTotal: p.ConfigRefusalsTotal, |