amadan

carlos / platform Public

console: loss paging backs off (hourly for the first day, then daily), runs in its own bounded worker, timestamps each attempt; copy HC5-N4..N7 approved by Paul

Paul's ruling 2026-10-12: back off rather than page hourly forever while
loss records have no acknowledgement. Answers Astra's two P2s on d918c363
and the record-write census entry for pageClaimLossesAt (key site,
discovery only).
Co-Authored-By: Codex <noreply@openai.com>
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
codex pushed by claude 435991b1acbe1ab0f8a3c646ddb6a80684b24c29 parent d918c36
10 files changed, +363 −34
  • internal/console/claim_authority_census_test.go +1 −1
  • internal/console/claim_loss_pages.go +46 −8
  • internal/console/claim_loss_pages_test.go +288 −10
  • internal/console/claim_runtime.go +1 −1
  • internal/console/server.go +2 −0
  • internal/console/store_gate.go +9 −0
  • internal/console/testdata/claim-authority-sites.txt +5 −3
  • internal/hostops/recovery.go +4 −6
  • internal/provision/plan9_b8_census_test.go +2 −0
  • testdata/claim-moves-copy.md +5 −5
diff --git a/internal/console/claim_authority_census_test.go b/internal/console/claim_authority_census_test.go
index 18161f0ab..156eb8ca8 100644
--- a/internal/console/claim_authority_census_test.go
+++ b/internal/console/claim_authority_census_test.go
@@ -75,7 +75,7 @@ func claimAuthoritySites(t *testing.T) []string {
}
// Loss paging reads signed control evidence and writes only its
// NonApp delivery journal. Inventory those sites explicitly.
- lossPageSite := e.Name() == "claim_loss_pages.go" && (method == "List" || method == "ReadLossRecord" || method == "Exec" || method == "ClaimLossPageAttempt")
+ lossPageSite := e.Name() == "claim_loss_pages.go" && (method == "List" || method == "ReadLossRecord" || method == "Exec" || method == "ExecContext" || method == "ClaimLossPageAttempt")
lossPageSchema := e.Name() == "store_gate.go" && fn.Name.Name == "OpenGated" && method == "Exec"
if !selected[method] && !lossPageSite && !lossPageSchema {
return true
diff --git a/internal/console/claim_loss_pages.go b/internal/console/claim_loss_pages.go
index 270600621..976e0198e 100644
--- a/internal/console/claim_loss_pages.go
+++ b/internal/console/claim_loss_pages.go
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "log"
"strings"
"time"
@@ -12,15 +13,46 @@ import (
"github.com/carlosframework/platform/internal/hostops"
)
-const claimLossPageInterval = time.Hour
+const (
+ claimLossPageInterval = time.Hour
+ claimLossPageDailyInterval = 24 * time.Hour
+ claimLossPageBudget = 30 * time.Second
+)
+
+// Recovery schedules one bounded scan. A slow sink holds only this worker;
+// later recovery passes skip scheduling until it exits.
+func (s *Server) scheduleClaimLossPages(ctx context.Context) {
+ if !s.cfg.HostClaim || ctx.Err() != nil {
+ return
+ }
+ s.claimLossPageMu.Lock()
+ defer s.claimLossPageMu.Unlock()
+ if s.claimLossPageDone != nil {
+ select {
+ case <-s.claimLossPageDone:
+ default:
+ return
+ }
+ }
+ done := make(chan struct{})
+ s.claimLossPageDone = done
+ go func() {
+ defer close(done)
+ pass, cancel := context.WithTimeout(ctx, claimLossPageBudget)
+ defer cancel()
+ if err := s.pageClaimLosses(pass); err != nil {
+ log.Printf("console: claim recovery: %v", err)
+ }
+ }()
+}
func (s *Server) pageClaimLosses(ctx context.Context) error {
- return s.pageClaimLossesAt(ctx, time.Now())
+ return s.pageClaimLossesAt(ctx, time.Now)
}
// Scan records independently of claims: loss evidence outlives a holder or
// operation, and may still need attention after its claim has gone.
-func (s *Server) pageClaimLossesAt(ctx context.Context, now time.Time) error {
+func (s *Server) pageClaimLossesAt(ctx context.Context, now func() time.Time) error {
if !s.cfg.HostClaim {
return nil
}
@@ -63,8 +95,11 @@ func (s *Server) pageClaimLossesAt(ctx context.Context, now time.Time) error {
return errors.Join(errs...)
}
-func (s *Server) pageClaimLoss(ctx context.Context, condition string, r hostclaim.LossRecord, now time.Time, subject, body string) error {
- attempted, err := s.store.ClaimLossPageAttempt(condition+" "+r.ID, now)
+func (s *Server) pageClaimLoss(ctx context.Context, condition string, r hostclaim.LossRecord, now func() time.Time, subject, body string) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ attempted, err := s.store.ClaimLossPageAttempt(ctx, condition+" "+r.ID, now())
if err != nil || !attempted {
return err
}
@@ -73,10 +108,13 @@ func (s *Server) pageClaimLoss(ctx context.Context, condition string, r hostclai
// ClaimLossPageAttempt reserves the attempt before delivery, including failed
// sends. The conditional write also serializes overlapping recovery passes.
-func (s *Store) ClaimLossPageAttempt(rule string, now time.Time) (bool, error) {
- result, err := s.db.Exec(`INSERT INTO claim_loss_pages (rule, attempted_at) VALUES (?, ?)
+func (s *Store) ClaimLossPageAttempt(ctx context.Context, rule string, now time.Time) (bool, error) {
+ result, err := s.db.ExecContext(ctx, `INSERT INTO claim_loss_pages (rule, first_attempted_at, attempted_at) VALUES (?, ?, ?)
ON CONFLICT(rule) DO UPDATE SET attempted_at = excluded.attempted_at
-WHERE claim_loss_pages.attempted_at <= ?`, rule, now.UnixNano(), now.Add(-claimLossPageInterval).UnixNano())
+WHERE claim_loss_pages.attempted_at <= CASE
+WHEN claim_loss_pages.first_attempted_at > ? THEN ? ELSE ? END`,
+ rule, now.UnixNano(), now.UnixNano(), now.Add(-claimLossPageDailyInterval).UnixNano(),
+ now.Add(-claimLossPageInterval).UnixNano(), now.Add(-claimLossPageDailyInterval).UnixNano())
if err != nil {
return false, err
}
diff --git a/internal/console/claim_loss_pages_test.go b/internal/console/claim_loss_pages_test.go
index bb6d42c38..6978fbd26 100644
--- a/internal/console/claim_loss_pages_test.go
+++ b/internal/console/claim_loss_pages_test.go
@@ -1,6 +1,7 @@
package console
import (
+ "context"
"crypto/ed25519"
"crypto/sha256"
"encoding/hex"
@@ -13,6 +14,7 @@ import (
"testing"
"time"
+ "github.com/aws/aws-sdk-go-v2/service/sns"
"github.com/carlosframework/platform/internal/alerting"
"github.com/carlosframework/platform/internal/hostclaim"
"github.com/carlosframework/platform/internal/release"
@@ -71,7 +73,7 @@ func TestClaimLossPagesIntervalAndRestart(t *testing.T) {
now := time.Date(2026, 10, 12, 13, 0, 0, 123, time.UTC)
pass := func() {
t.Helper()
- if err := s.pageClaimLossesAt(t.Context(), now); err != nil {
+ if err := s.pageClaimLossesAt(t.Context(), func() time.Time { return now }); err != nil {
t.Fatal(err)
}
}
@@ -119,7 +121,7 @@ func TestClaimLossPagesClearedAndTimers(t *testing.T) {
r := lossPageRecord("loss-timer", false, knowledge)
putLossPageRecord(t, raw, signer, r)
now := r.AcceptedAt
- if err := s.pageClaimLossesAt(t.Context(), now); err != nil {
+ if err := s.pageClaimLossesAt(t.Context(), func() time.Time { return now }); err != nil {
t.Fatal(err)
}
want := 1
@@ -136,7 +138,7 @@ func TestClaimLossPagesClearedAndTimers(t *testing.T) {
if err := raw.Delete(t.Context(), key); err != nil {
t.Fatal(err)
}
- if err := s.pageClaimLossesAt(t.Context(), now.Add(2*time.Hour)); err != nil {
+ if err := s.pageClaimLossesAt(t.Context(), func() time.Time { return now.Add(2 * time.Hour) }); err != nil {
t.Fatal(err)
}
if len(*pages) != want {
@@ -155,6 +157,7 @@ func TestClaimLossPagesRecoveryWiringWithoutMovesOrClaims(t *testing.T) {
if err := s.RunClaimRecoveryPass(t.Context()); err != nil {
t.Fatal(err)
}
+ waitClaimLossPages(t, s)
if len(*pages) != 1 {
t.Fatalf("record without a claim was not paged: %v", *pages)
}
@@ -165,12 +168,16 @@ func TestClaimLossPagesFlagOff(t *testing.T) {
alerting.Arm(&alerting.Sinks{WebhookURL: "http://unused", Post: func(string, []byte) error { calls++; return nil }})
t.Cleanup(func() { alerting.Arm(nil) })
s := &Server{}
- if err := s.pageClaimLossesAt(t.Context(), time.Now()); err != nil {
+ if err := s.pageClaimLossesAt(t.Context(), time.Now); err != nil {
t.Fatal(err)
}
if err := s.RunClaimRecoveryPass(t.Context()); err != nil {
t.Fatal(err)
}
+ s.scheduleClaimLossPages(t.Context())
+ if s.claimLossPageDone != nil {
+ t.Fatal("disabled runtime started a worker")
+ }
if calls != 0 {
t.Fatal("disabled runtime paged")
}
@@ -192,10 +199,10 @@ func TestClaimLossPageFailedDeliveryIsRateLimited(t *testing.T) {
failure := errors.New("delivery failed")
alerting.Arm(&alerting.Sinks{WebhookURL: "http://unused", Post: func(string, []byte) error { calls++; return failure }})
now := time.Now()
- if err := s.pageClaimLossesAt(t.Context(), now); !errors.Is(err, failure) {
+ if err := s.pageClaimLossesAt(t.Context(), func() time.Time { return now }); !errors.Is(err, failure) {
t.Fatalf("delivery: %v", err)
}
- if err := s.pageClaimLossesAt(t.Context(), now.Add(time.Minute)); err != nil {
+ if err := s.pageClaimLossesAt(t.Context(), func() time.Time { return now.Add(time.Minute) }); err != nil {
t.Fatal(err)
}
if calls != 1 {
@@ -222,6 +229,7 @@ func TestClaimLossPagesAutomaticReportingDoesNotAcknowledge(t *testing.T) {
if err := s.RunClaimRecoveryPass(t.Context()); err != nil {
t.Fatal(err)
}
+ waitClaimLossPages(t, s)
if len(*pages) != 2 {
t.Fatalf("materialized record must page both conditions in the same pass: %v", *pages)
}
@@ -229,7 +237,7 @@ func TestClaimLossPagesAutomaticReportingDoesNotAcknowledge(t *testing.T) {
if err != nil || !c.Claim.EvidenceRefs[0].Reported {
t.Fatalf("reporting did not finish: %+v %v", c, err)
}
- if err := s.pageClaimLossesAt(t.Context(), time.Now().Add(2*time.Hour)); err != nil {
+ if err := s.pageClaimLossesAt(t.Context(), func() time.Time { return time.Now().Add(2 * time.Hour) }); err != nil {
t.Fatal(err)
}
if len(*pages) != 4 {
@@ -246,7 +254,7 @@ func TestClaimLossPageReservationSerializesPasses(t *testing.T) {
wg.Add(1)
go func() {
defer wg.Done()
- ok, err := st.ClaimLossPageAttempt("pending loss", now)
+ ok, err := st.ClaimLossPageAttempt(t.Context(), "pending loss", now)
if err != nil {
t.Error(err)
}
@@ -267,7 +275,7 @@ func TestClaimLossPagesRejectBadEvidenceAndContinue(t *testing.T) {
t.Fatal(err)
}
putLossPageRecord(t, raw, signer, lossPageRecord("valid", true, hostclaim.TimersComplete))
- if err := s.pageClaimLossesAt(t.Context(), time.Now()); !errors.Is(err, hostclaim.ErrLossRecord) {
+ if err := s.pageClaimLossesAt(t.Context(), time.Now); !errors.Is(err, hostclaim.ErrLossRecord) {
t.Fatalf("invalid signature accepted: %v", err)
}
if len(*pages) != 1 {
@@ -281,10 +289,280 @@ func TestClaimLossPagesJournalFailureDoesNotSend(t *testing.T) {
if err := s.store.Close(); err != nil {
t.Fatal(err)
}
- if err := s.pageClaimLossesAt(t.Context(), time.Now()); err == nil {
+ if err := s.pageClaimLossesAt(t.Context(), time.Now); err == nil {
t.Fatal("journal error hidden")
}
if len(*pages) != 0 {
t.Fatal("sent without durable reservation")
}
}
+
+func waitClaimLossPages(t *testing.T, s *Server) {
+ t.Helper()
+ s.claimLossPageMu.Lock()
+ done := s.claimLossPageDone
+ s.claimLossPageMu.Unlock()
+ if done == nil {
+ t.Fatal("no loss paging worker scheduled")
+ }
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("loss paging worker did not finish")
+ }
+}
+
+func TestClaimLossPagesHourlyToDailyAcrossReopen(t *testing.T) {
+ s, raw, signer, pages := lossPageWorld(t)
+ r := lossPageRecord("old-loss", true, hostclaim.TimersUnknown)
+ putLossPageRecord(t, raw, signer, r)
+ first := r.AcceptedAt.Add(30 * 24 * time.Hour)
+ pass := func(at time.Time, want int) {
+ t.Helper()
+ if err := s.pageClaimLossesAt(t.Context(), func() time.Time { return at }); err != nil {
+ t.Fatal(err)
+ }
+ if len(*pages) != want {
+ t.Fatalf("at %s: got %d pages, want %d", at.Sub(first), len(*pages), want)
+ }
+ }
+ for hour := 0; hour < 24; hour++ {
+ pass(first.Add(time.Duration(hour)*time.Hour), 2*(hour+1))
+ }
+ path := s.store.gate.path
+ if err := s.store.Close(); err != nil {
+ t.Fatal(err)
+ }
+ st, err := OpenGated(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { st.Close() })
+ s = &Server{cfg: s.cfg, store: st}
+ pass(first.Add(24*time.Hour-time.Nanosecond), 48)
+ // Once the first 24 hours end, the last attempt must be a day old.
+ pass(first.Add(24*time.Hour), 48)
+ pass(first.Add(47*time.Hour-time.Nanosecond), 48)
+ pass(first.Add(47*time.Hour), 50)
+ pass(first.Add(71*time.Hour-time.Nanosecond), 50)
+ pass(first.Add(71*time.Hour), 52)
+ for _, condition := range []string{"pending", "timers"} {
+ var saved int64
+ if err := st.db.QueryRow(`SELECT first_attempted_at FROM claim_loss_pages WHERE rule = ?`, condition+" "+r.ID).Scan(&saved); err != nil || saved != first.UnixNano() {
+ t.Fatalf("first %s attempt changed: %d %v", condition, saved, err)
+ }
+ }
+}
+
+func TestClaimLossPagesClockAtEachReservation(t *testing.T) {
+ s, raw, signer, _ := lossPageWorld(t)
+ for _, id := range []string{"a-early", "b-late"} {
+ putLossPageRecord(t, raw, signer, lossPageRecord(id, true, hostclaim.TimersComplete))
+ }
+ first := time.Now()
+ now := first
+ late := 0
+ calls := 0
+ alerting.Arm(&alerting.Sinks{WebhookURL: "http://unused", Post: func(_ string, b []byte) error {
+ calls++
+ if strings.Contains(string(b), "b-late") {
+ late++
+ }
+ if calls == 1 {
+ now = now.Add(time.Hour)
+ }
+ return nil
+ }})
+ clock := func() time.Time { return now }
+ if err := s.pageClaimLossesAt(t.Context(), clock); err != nil {
+ t.Fatal(err)
+ }
+ now = now.Add(10 * time.Second)
+ if err := s.pageClaimLossesAt(t.Context(), clock); err != nil {
+ t.Fatal(err)
+ }
+ if late != 1 || calls != 3 {
+ t.Fatalf("late record retried ten seconds later: late=%d, total=%d", late, calls)
+ }
+ var reserved int64
+ if err := s.store.db.QueryRow(`SELECT attempted_at FROM claim_loss_pages WHERE rule = 'pending b-late'`).Scan(&reserved); err != nil || reserved != first.Add(time.Hour).UnixNano() {
+ t.Fatalf("late reservation: %d %v", reserved, err)
+ }
+}
+
+func TestClaimLossPagesBlockedSinkDoesNotBlockRecovery(t *testing.T) {
+ s, raw, signer, _ := lossPageWorld(t)
+ putLossPageRecord(t, raw, signer, lossPageRecord("blocking", true, hostclaim.TimersComplete))
+ entered, releaseSink := make(chan struct{}), make(chan struct{})
+ var unblock sync.Once
+ var calls atomic.Int64
+ alerting.Arm(&alerting.Sinks{WebhookURL: "http://unused", Post: func(string, []byte) error {
+ if calls.Add(1) == 1 {
+ close(entered)
+ <-releaseSink
+ }
+ return nil
+ }})
+ t.Cleanup(func() {
+ unblock.Do(func() { close(releaseSink) })
+ waitClaimLossPages(t, s)
+ })
+ var records, purges, admissions atomic.Int64
+ wrap := func(fn func(context.Context) error, count *atomic.Int64) func(context.Context) error {
+ return func(ctx context.Context) error { count.Add(1); return fn(ctx) }
+ }
+ r := s.cfg.HostClaimRecovery
+ r.SweepRecords = wrap(r.SweepRecords, &records)
+ r.SweepPurges = wrap(r.SweepPurges, &purges)
+ r.SweepAdmissions = wrap(r.SweepAdmissions, &admissions)
+ pass := func() {
+ t.Helper()
+ done := make(chan error, 1)
+ go func() { done <- s.RunClaimRecoveryPass(t.Context()) }()
+ select {
+ case err := <-done:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(2 * time.Second):
+ unblock.Do(func() { close(releaseSink) })
+ <-done
+ t.Fatal("paging blocked recovery")
+ }
+ }
+ pass()
+ select {
+ case <-entered:
+ case <-time.After(2 * time.Second):
+ t.Fatal("sink was not called")
+ }
+ s.claimLossPageMu.Lock()
+ worker := s.claimLossPageDone
+ s.claimLossPageMu.Unlock()
+ for range 3 {
+ pass()
+ }
+ if records.Load() != 4 || purges.Load() != 4 || admissions.Load() != 4 {
+ t.Fatalf("sweeps delayed: records=%d purges=%d admissions=%d", records.Load(), purges.Load(), admissions.Load())
+ }
+ s.claimLossPageMu.Lock()
+ overlapped := s.claimLossPageDone != worker
+ s.claimLossPageMu.Unlock()
+ if overlapped || calls.Load() != 1 {
+ t.Fatal("paging workers overlapped")
+ }
+ unblock.Do(func() { close(releaseSink) })
+ waitClaimLossPages(t, s)
+ putLossPageRecord(t, raw, signer, lossPageRecord("next-worker", true, hostclaim.TimersComplete))
+ pass()
+ waitClaimLossPages(t, s)
+ if calls.Load() != 2 {
+ t.Fatal("next recovery did not schedule a new paging worker")
+ }
+}
+
+func TestClaimLossPageJournalUpgrade(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "old.db")
+ st, err := Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ at := time.Now().Add(-23 * time.Hour)
+ _, err = st.db.Exec(`CREATE TABLE claim_loss_pages (rule TEXT PRIMARY KEY, attempted_at INTEGER NOT NULL);
+INSERT INTO claim_loss_pages VALUES ('pending old', ?)`, at.UnixNano())
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := st.Close(); err != nil {
+ t.Fatal(err)
+ }
+ for range 2 {
+ st, err := OpenGated(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var first, last int64
+ err = st.db.QueryRow(`SELECT first_attempted_at, attempted_at FROM claim_loss_pages WHERE rule = 'pending old'`).Scan(&first, &last)
+ st.Close()
+ if err != nil || first != at.UnixNano() || last != at.UnixNano() {
+ t.Fatalf("upgrade lost durable attempt: %d %d %v", first, last, err)
+ }
+ }
+}
+
+type lossPageDeadlineStore struct {
+ release.Store
+ deadline chan time.Time
+}
+
+func (s lossPageDeadlineStore) List(ctx context.Context, prefix string) ([]string, error) {
+ deadline, _ := ctx.Deadline()
+ s.deadline <- deadline
+ <-ctx.Done()
+ return nil, ctx.Err()
+}
+
+func TestClaimLossPagesWorkerBudget(t *testing.T) {
+ s, _, _, pages := lossPageWorld(t)
+ deadline := make(chan time.Time, 1)
+ s.cfg.Releases = lossPageDeadlineStore{Store: s.cfg.Releases, deadline: deadline}
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ before := time.Now()
+ s.scheduleClaimLossPages(ctx)
+ select {
+ case at := <-deadline:
+ if at.Before(before.Add(claimLossPageBudget)) || at.After(time.Now().Add(claimLossPageBudget)) {
+ t.Errorf("scan has no per-pass budget: deadline=%v", at)
+ }
+ case <-time.After(time.Second):
+ t.Error("worker never started discovery")
+ }
+ cancel()
+ waitClaimLossPages(t, s)
+ if len(*pages) != 0 {
+ t.Fatal("canceled scan delivered a page")
+ }
+ s.claimLossPageMu.Lock()
+ done := s.claimLossPageDone
+ s.claimLossPageMu.Unlock()
+ s.scheduleClaimLossPages(ctx)
+ if s.claimLossPageDone != done {
+ t.Fatal("canceled recovery scheduled another worker")
+ }
+}
+
+type lossPageBlockingSNS struct {
+ entered chan time.Time
+}
+
+func (p lossPageBlockingSNS) Publish(ctx context.Context, _ *sns.PublishInput, _ ...func(*sns.Options)) (*sns.PublishOutput, error) {
+ deadline, _ := ctx.Deadline()
+ p.entered <- deadline
+ <-ctx.Done()
+ return nil, ctx.Err()
+}
+
+func TestClaimLossPagesSinkTimeoutAndCancellation(t *testing.T) {
+ s, raw, signer, _ := lossPageWorld(t)
+ putLossPageRecord(t, raw, signer, lossPageRecord("blocked-sns", true, hostclaim.TimersUnknown))
+ entered := make(chan time.Time, 1)
+ alerting.Arm(&alerting.Sinks{TopicARN: "test", Pub: lossPageBlockingSNS{entered: entered}})
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ s.scheduleClaimLossPages(ctx)
+ select {
+ case deadline := <-entered:
+ if deadline.IsZero() || time.Until(deadline) > alerting.PublishTimeout {
+ t.Errorf("SNS sink has no timeout: %v", deadline)
+ }
+ case <-time.After(time.Second):
+ t.Error("worker never called SNS")
+ }
+ cancel()
+ waitClaimLossPages(t, s)
+ var attempts int
+ if err := s.store.db.QueryRow(`SELECT count(*) FROM claim_loss_pages`).Scan(&attempts); err != nil || attempts != 1 {
+ t.Fatalf("canceled pass reserved another condition: %d %v", attempts, err)
+ }
+}
diff --git a/internal/console/claim_runtime.go b/internal/console/claim_runtime.go
index b606fa928..75f3f0bc0 100644
--- a/internal/console/claim_runtime.go
+++ b/internal/console/claim_runtime.go
@@ -127,7 +127,7 @@ func (s *Server) ConfigureClaimRuntime(ctx context.Context) error {
moveLogs = s.drainMoveLog
}
s.cfg.HostClaimRecovery = &hostops.Recovery{
- LossPages: s.pageClaimLosses,
+ ScheduleLossPages: s.scheduleClaimLossPages,
ReportAutomaticLosses: func(ctx context.Context, host string) error {
return a.Control.ReportAutomaticLosses(ctx, a.Signer, host)
},
diff --git a/internal/console/server.go b/internal/console/server.go
index 7a60ea992..da8ae45bd 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -821,6 +821,8 @@ type Config struct {
// Server is the console's http.Handler.
type Server struct {
+ claimLossPageMu sync.Mutex
+ claimLossPageDone chan struct{}
// Enabled claim runtime (ConfigureClaimRuntime). Unset flag-off.
claimRuntimeConfigured bool
claimOwner hostops.RegistrationOwner
diff --git a/internal/console/store_gate.go b/internal/console/store_gate.go
index 134c9b725..370b5475b 100644
--- a/internal/console/store_gate.go
+++ b/internal/console/store_gate.go
@@ -1792,10 +1792,19 @@ func OpenGated(path string) (*Store, error) {
}
if _, err := s.db.Exec(`CREATE TABLE IF NOT EXISTS claim_loss_pages (
rule TEXT PRIMARY KEY,
+ first_attempted_at INTEGER NOT NULL,
attempted_at INTEGER NOT NULL
);`); err != nil {
return err
}
+ // The earlier journal retained only the last attempt. Preserve that
+ // durable time as the start for existing rows; record age is irrelevant.
+ if _, err := s.db.Exec(`ALTER TABLE claim_loss_pages ADD COLUMN first_attempted_at INTEGER`); err != nil && !strings.Contains(err.Error(), "duplicate column") {
+ return err
+ }
+ if _, err := s.db.Exec(`UPDATE claim_loss_pages SET first_attempted_at = attempted_at WHERE first_attempted_at IS NULL`); err != nil {
+ return err
+ }
return s.migrateClaimAppIdentity()
})
}
diff --git a/internal/console/testdata/claim-authority-sites.txt b/internal/console/testdata/claim-authority-sites.txt
index d4c863cb4..ed4ae56ac 100644
--- a/internal/console/testdata/claim-authority-sites.txt
+++ b/internal/console/testdata/claim-authority-sites.txt
@@ -125,8 +125,10 @@ store_gate.go|openGated|sql.OpenDB(claimConnector{g: g, trusted: true})
store_gate.go|openGated|sql.OpenDB(claimConnector{g: g})
store_gate.go|withClaimSQL|s.withPreparedClaimSQL(ctx, scopes, check, func(ctx context.Context, _ claimSQLScope) error { return ctx.Err() }, nil, false, fn)
view.go|openRegistry|registry.Open(path)
-claim_loss_pages.go|ClaimLossPageAttempt|s.db.Exec(`INSERT INTO claim_loss_pages (rule, attempted_at) VALUES (?, ?) ON CONFLICT(rule) DO UPDATE SET attempted_at = excluded.attempted_at WHERE claim_loss_pages.attempted_at <= ?`, rule, now.UnixNano(), now.Add(-claimLossPageInterval).UnixNano())
+claim_loss_pages.go|ClaimLossPageAttempt|s.db.ExecContext(ctx, `INSERT INTO claim_loss_pages (rule, first_attempted_at, attempted_at) VALUES (?, ?, ?) ON CONFLICT(rule) DO UPDATE SET attempted_at = excluded.attempted_at WHERE claim_loss_pages.attempted_at <= CASE WHEN claim_loss_pages.first_attempted_at > ? THEN ? ELSE ? END`, rule, now.UnixNano(), now.UnixNano(), now.Add(-claimLossPageDailyInterval).UnixNano(), now.Add(-claimLossPageInterval).UnixNano(), now.Add(-claimLossPageDailyInterval).UnixNano())
claim_loss_pages.go|pageClaimLossesAt|hostclaim.ReadLossRecord(ctx, s.cfg.Releases, s.cfg.ReleasePub, id)
claim_loss_pages.go|pageClaimLossesAt|s.cfg.Releases.List(ctx, "control/host-loss/")
-claim_loss_pages.go|pageClaimLoss|s.store.ClaimLossPageAttempt(condition+" "+r.ID, now)
-store_gate.go|OpenGated|s.db.Exec(`CREATE TABLE IF NOT EXISTS claim_loss_pages ( rule TEXT PRIMARY KEY, attempted_at INTEGER NOT NULL );`)
+claim_loss_pages.go|pageClaimLoss|s.store.ClaimLossPageAttempt(ctx, condition+" "+r.ID, now())
+store_gate.go|OpenGated|s.db.Exec(`CREATE TABLE IF NOT EXISTS claim_loss_pages ( rule TEXT PRIMARY KEY, first_attempted_at INTEGER NOT NULL, attempted_at INTEGER NOT NULL );`)
+store_gate.go|OpenGated|s.db.Exec(`ALTER TABLE claim_loss_pages ADD COLUMN first_attempted_at INTEGER`)
+store_gate.go|OpenGated|s.db.Exec(`UPDATE claim_loss_pages SET first_attempted_at = attempted_at WHERE first_attempted_at IS NULL`)
diff --git a/internal/hostops/recovery.go b/internal/hostops/recovery.go
index a92885a9b..64d02ef1d 100644
--- a/internal/hostops/recovery.go
+++ b/internal/hostops/recovery.go
@@ -38,8 +38,8 @@ func ListClaims(ctx context.Context, s release.Store) ([]string, error) {
}
type Recovery struct {
- // LossPages scans retained loss records once per pass, after materialization.
- LossPages func(context.Context) error
+ // ScheduleLossPages triggers a separate worker after materialization.
+ ScheduleLossPages func(context.Context)
ReportAutomaticLosses func(context.Context, string) error
MoveFenceAuthority hostclaim.FenceAuthority
Moves func(context.Context, *MoveRecoveryGrant) error
@@ -123,10 +123,8 @@ func (r Recovery) Pass(ctx context.Context) error {
all = append(all, err)
}
}
- if r.LossPages != nil {
- if err := r.LossPages(ctx); err != nil {
- all = append(all, err)
- }
+ if r.ScheduleLossPages != nil {
+ r.ScheduleLossPages(ctx)
}
if r.MoveLogs != nil {
if err := runMoveLogDrain(ctx, r.MoveLogs); err != nil && !errors.Is(err, ErrPending) {
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go
index 99f4fac58..b240c3b12 100644
--- a/internal/provision/plan9_b8_census_test.go
+++ b/internal/provision/plan9_b8_census_test.go
@@ -472,6 +472,8 @@ func Reviewed(ctx context.Context, s Store, k string) error {
// BEGIN REVIEWED INVENTORY
var plan9B8CensusInventory = []censusEntry{
+ // Read-only discovery of signed loss records; no record write or lifecycle authority.
+ {"internal/console/claim_loss_pages.go", "Server.pageClaimLossesAt", "key-site", "key:LossRecordKey", "c60f826092bf31e7f657ec60592187f433026e4a53fb5d9ca011a696d7cd2134"},
{"internal/hostbox/parkleases.go", "ReplicaParkLeaseSet", "key-site", "lit:instances", "0ca8d9b743402fda9d15cc215464423b35e21c1c2b47f91294d76188b7d43fb6"},
{"internal/host/claimretirementcensus.go", "ClaimRetirementCensus", "key-site", "lit:instances", "abc0bd8c5f5d328abfac8821f42f8f23954723ed6506747f33b0197824e2611f"},
{"internal/host/claimproduction.go", "productionClaimRegional.View", "key-site", "lit:instances", "533203b2aa729801438da4f3a85799a58ea0e67d20eb5625f0de32ea623f1607"},
diff --git a/testdata/claim-moves-copy.md b/testdata/claim-moves-copy.md
index 532c335e0..179c4560b 100644
--- a/testdata/claim-moves-copy.md
+++ b/testdata/claim-moves-copy.md
@@ -1,4 +1,4 @@
-# Claim moves copy record (HC5-N1 approved by Paul 2026-10-08; HC5-N2 and HC5-N3, from claim Plan 10, approved by Paul 2026-10-10)
+# Claim moves copy record (HC5-N1 approved by Paul 2026-10-08; HC5-N2 and HC5-N3, from claim Plan 10, approved by Paul 2026-10-10; HC5-N4 to HC5-N7, claim loss pages, approved by Paul 2026-10-12)
| id | file | text | status |
|---|---|---|---|
@@ -10,7 +10,7 @@
| id | file | text | status |
|---|---|---|---|
-| HC5-N4 | internal/console/claim_loss_pages.go | Loss recorded for <host> | pending |
-| HC5-N5 | internal/console/claim_loss_pages.go | A loss was recorded for host <host>. Review loss record <id> and check what needs recovery. | pending |
-| HC5-N6 | internal/console/claim_loss_pages.go | Timers may be lost for <host> | pending |
-| HC5-N7 | internal/console/claim_loss_pages.go | Timers may have been lost with the box serving host <host>. Check loss record <id>. Check which timers are missing and re-arm them. | pending |
+| HC5-N4 | internal/console/claim_loss_pages.go | Loss recorded for <host> | approved |
+| HC5-N5 | internal/console/claim_loss_pages.go | A loss was recorded for host <host>. Review loss record <id> and check what needs recovery. | approved |
+| HC5-N6 | internal/console/claim_loss_pages.go | Timers may be lost for <host> | approved |
+| HC5-N7 | internal/console/claim_loss_pages.go | Timers may have been lost with the box serving host <host>. Check loss record <id>. Check which timers are missing and re-arm them. | approved |