| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "database/sql" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "log" |
| 9 | "sort" |
| 10 | "strings" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/alerting" |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | "github.com/carlosframework/platform/internal/hostops" |
| 16 | ) |
| 17 | |
| 18 | const ( |
| 19 | claimLossPageInterval = time.Hour |
| 20 | claimLossPageDailyInterval = 24 * time.Hour |
| 21 | claimLossPageBudget = 30 * time.Second |
| 22 | ) |
| 23 | |
| 24 | // claimLossReadBudget bounds one record's read inside a scan. A record whose |
| 25 | // read keeps stalling then fails on its own and the scan moves past it, |
| 26 | // retrying it next lap, instead of spending the whole scan budget on it every |
| 27 | // time and starving the records behind it. |
| 28 | var claimLossReadBudget = 5 * time.Second |
| 29 | |
| 30 | // Recovery schedules one bounded scan. A slow sink holds only this worker; |
| 31 | // later recovery passes skip scheduling until it exits. |
| 32 | func (s *Server) scheduleClaimLossPages(ctx context.Context) { |
| 33 | if !s.cfg.HostClaim || ctx.Err() != nil { |
| 34 | return |
| 35 | } |
| 36 | s.claimLossPageMu.Lock() |
| 37 | defer s.claimLossPageMu.Unlock() |
| 38 | if s.claimLossPageDone != nil { |
| 39 | select { |
| 40 | case <-s.claimLossPageDone: |
| 41 | default: |
| 42 | return |
| 43 | } |
| 44 | } |
| 45 | done := make(chan struct{}) |
| 46 | s.claimLossPageDone = done |
| 47 | go func() { |
| 48 | defer close(done) |
| 49 | pass, cancel := context.WithTimeout(ctx, claimLossPageBudget) |
| 50 | defer cancel() |
| 51 | if err := s.pageClaimLosses(pass); err != nil { |
| 52 | log.Printf("console: claim recovery: %v", err) |
| 53 | } |
| 54 | }() |
| 55 | } |
| 56 | |
| 57 | func (s *Server) pageClaimLosses(ctx context.Context) error { |
| 58 | return s.pageClaimLossesAt(ctx, time.Now) |
| 59 | } |
| 60 | |
| 61 | // Scan records independently of claims: loss evidence outlives a holder or |
| 62 | // operation, and may still need attention after its claim has gone. |
| 63 | func (s *Server) pageClaimLossesAt(ctx context.Context, now func() time.Time) error { |
| 64 | if !s.cfg.HostClaim { |
| 65 | return nil |
| 66 | } |
| 67 | if s.cfg.Releases == nil || s.store == nil || len(s.cfg.ReleasePub) == 0 { |
| 68 | return hostops.ErrContract |
| 69 | } |
| 70 | var after string |
| 71 | if err := s.store.db.QueryRowContext(ctx, `SELECT last_key FROM claim_loss_pages WHERE rule = 'scan'`).Scan(&after); err != nil && !errors.Is(err, sql.ErrNoRows) { |
| 72 | return err |
| 73 | } |
| 74 | keys, err := s.cfg.Releases.List(ctx, "control/host-loss/") |
| 75 | if err != nil { |
| 76 | return err |
| 77 | } |
| 78 | // Store listings need not be ordered. Visit each key once, wrapping even |
| 79 | // when the saved key has since been removed or earlier keys were added. |
| 80 | sort.Strings(keys) |
| 81 | start := sort.Search(len(keys), func(i int) bool { return keys[i] > after }) |
| 82 | var errs []error |
| 83 | for i := range keys { |
| 84 | if err := ctx.Err(); err != nil { |
| 85 | return errors.Join(append(errs, err)...) |
| 86 | } |
| 87 | key := keys[(start+i)%len(keys)] |
| 88 | errs = append(errs, s.pageClaimLossRecord(ctx, key, now)) |
| 89 | if err := ctx.Err(); err != nil { |
| 90 | return errors.Join(append(errs, err)...) |
| 91 | } |
| 92 | // Never advance past canceled work. Other record errors are retried on |
| 93 | // the next lap so one bad record cannot block everything after it. |
| 94 | if _, err := s.store.db.ExecContext(ctx, `INSERT INTO claim_loss_pages (rule, first_attempted_at, attempted_at, last_key) VALUES ('scan', 0, 0, ?) |
| 95 | ON CONFLICT(rule) DO UPDATE SET last_key = excluded.last_key`, key); err != nil { |
| 96 | return errors.Join(append(errs, err)...) |
| 97 | } |
| 98 | } |
| 99 | return errors.Join(errs...) |
| 100 | } |
| 101 | |
| 102 | func (s *Server) pageClaimLossRecord(ctx context.Context, key string, now func() time.Time) error { |
| 103 | id := strings.TrimSuffix(strings.TrimPrefix(key, "control/host-loss/"), ".json") |
| 104 | exact, err := hostclaim.LossRecordKey(id) |
| 105 | if err != nil || exact != key { |
| 106 | return hostclaim.ErrLossRecord |
| 107 | } |
| 108 | read, cancel := context.WithTimeout(ctx, claimLossReadBudget) |
| 109 | observed, err := hostclaim.ReadLossRecord(read, s.cfg.Releases, s.cfg.ReleasePub, id) |
| 110 | cancel() |
| 111 | if errors.Is(err, hostclaim.ErrLossRecordAbsent) { |
| 112 | return nil |
| 113 | } |
| 114 | if err != nil { |
| 115 | return err |
| 116 | } |
| 117 | r := observed.Record |
| 118 | var errs []error |
| 119 | if r.AlertsPending { |
| 120 | subject, body := claimLossPendingPage(r.Host, r.ID) |
| 121 | errs = append(errs, s.pageClaimLoss(ctx, "pending", r, now, subject, body)) |
| 122 | } |
| 123 | if r.Timers.Knowledge == hostclaim.TimersPartial || r.Timers.Knowledge == hostclaim.TimersUnknown { |
| 124 | subject, body := claimTimerLossPage(r.Host, r.ID) |
| 125 | errs = append(errs, s.pageClaimLoss(ctx, "timers", r, now, subject, body)) |
| 126 | } |
| 127 | return errors.Join(errs...) |
| 128 | } |
| 129 | |
| 130 | func (s *Server) pageClaimLoss(ctx context.Context, condition string, r hostclaim.LossRecord, now func() time.Time, subject, body string) error { |
| 131 | if err := ctx.Err(); err != nil { |
| 132 | return err |
| 133 | } |
| 134 | attempted, err := s.store.ClaimLossPageAttempt(ctx, condition+" "+r.ID, now()) |
| 135 | if err != nil || !attempted { |
| 136 | return err |
| 137 | } |
| 138 | return alerting.Page(ctx, alerting.LevelWarning, subject, body) |
| 139 | } |
| 140 | |
| 141 | // ClaimLossPageAttempt reserves the attempt before delivery, including failed |
| 142 | // sends. The conditional write also serializes overlapping recovery passes. |
| 143 | func (s *Store) ClaimLossPageAttempt(ctx context.Context, rule string, now time.Time) (bool, error) { |
| 144 | result, err := s.db.ExecContext(ctx, `INSERT INTO claim_loss_pages (rule, first_attempted_at, attempted_at) VALUES (?, ?, ?) |
| 145 | ON CONFLICT(rule) DO UPDATE SET attempted_at = excluded.attempted_at |
| 146 | WHERE claim_loss_pages.attempted_at <= CASE |
| 147 | WHEN claim_loss_pages.first_attempted_at > ? THEN ? ELSE ? END`, |
| 148 | rule, now.UnixNano(), now.UnixNano(), now.Add(-claimLossPageDailyInterval).UnixNano(), |
| 149 | now.Add(-claimLossPageInterval).UnixNano(), now.Add(-claimLossPageDailyInterval).UnixNano()) |
| 150 | if err != nil { |
| 151 | return false, err |
| 152 | } |
| 153 | n, err := result.RowsAffected() |
| 154 | return n == 1, err |
| 155 | } |
| 156 | |
| 157 | // Draft operator copy. The controller collects these strings for approval. |
| 158 | func claimLossPendingPage(host, id string) (subject, body string) { |
| 159 | return fmt.Sprintf("Loss recorded for %s", host), |
| 160 | fmt.Sprintf("A loss was recorded for host %s. Review loss record %s and check what needs recovery.", host, id) |
| 161 | } |
| 162 | |
| 163 | func claimTimerLossPage(host, id string) (subject, body string) { |
| 164 | return fmt.Sprintf("Timers may be lost for %s", host), |
| 165 | fmt.Sprintf("Timers may have been lost with the box serving host %s. Check loss record %s. Check which timers are missing and re-arm them.", host, id) |
| 166 | } |
| 167 | |
amadan