amadan

carlos / platform Public

console: the reclaim answers held, unreadable or reclaimed; two lost races are busy

Fix round 1, Important 1: offlineReclaimable returns a three-way verdict.
A holder record that cannot be read, a corrupt body, a fence that fails,
or a row or cleanup evidence read that fails is unreadable, today's 502
"could not read the instance", byte for byte with the one-move lock's
answer; positive evidence of a live holder (the record carries the move,
a pending row or issued request, an unsettled cleanup, a pending lifetime
operation on the host) is busy; only a clean negative after the fence
reclaims. Minor 6: after two lost races on active.json the answer is busy,
naming the holder a last read finds, else the reserver's own host.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
claude 27164dcba6bc54d0a456bb817eab1779fa80b57a parent 714ca77
1 file changed, +77 −25
  • internal/console/offlinereserve.go +77 −25
diff --git a/internal/console/offlinereserve.go b/internal/console/offlinereserve.go
index 1a8ee33aa..eba573825 100644
--- a/internal/console/offlinereserve.go
+++ b/internal/console/offlinereserve.go
@@ -248,15 +248,28 @@ func (s *Server) reserveBoxes(ctx context.Context, sel offlineSelection, r moves
return offlineReserveUnreadable
}
}
+ // Two lost races: busy. When neither read saw a holder (the object
+ // was created under both of them), one more read names it; failing
+ // that, the sentence names the reserver's own host, which is moving
+ // nowhere yet but is the one the caller asked about.
if lastHolder == "" {
- return offlineReserveUnreadable
+ if raw, _, err := s.cfg.Releases.Get(ctx, moves.OfflineActiveKey); err == nil {
+ if table, err := moves.ParseOfflineActive(raw); err == nil {
+ lastHolder = moves.OfflineActiveBody(table).Host
+ }
+ }
+ }
+ if lastHolder == "" {
+ lastHolder = r.Host
}
return offlineBusy(lastHolder)
}
// offlineInTheWay answers busy, naming the oldest holder, unless every
// conflicting operation is reclaimable. A young operation is busy without
-// any reclaim being attempted, so nothing is fenced while one is.
+// any reclaim being attempted, so nothing is fenced while one is. An old
+// one the reclaim rule could not judge (reclaimUnreadable) is the 502 the
+// one-move lock answered for an unreadable holder record, byte for byte.
func (s *Server) offlineInTheWay(ctx context.Context, conflicts map[offlineOp]map[string]moves.OfflineReservation) *offlineRefusal {
type held struct {
op offlineOp
@@ -286,16 +299,37 @@ func (s *Server) offlineInTheWay(ctx context.Context, conflicts map[offlineOp]ma
}
}
for _, h := range ops {
- if !s.offlineReclaimable(ctx, h.entries) {
+ switch s.offlineReclaimable(ctx, h.entries) {
+ case reclaimHeld:
return offlineBusy(h.first.Host)
+ case reclaimUnreadable:
+ return offlineReserveUnreadable
}
}
return nil
}
+// reclaimVerdict is the reclaim rule's answer for one operation.
+type reclaimVerdict int
+
+const (
+ // reclaimHeld: positive evidence the operation is live (its record
+ // carries the move, its row is pending or has a request issued, its
+ // cleanup is unsettled, a lifetime operation on its host is pending).
+ // The reserver answers busy.
+ reclaimHeld reclaimVerdict = iota
+ // reclaimUnreadable: the evidence could not be read (a store read, a
+ // corrupt body, a fence that failed, a row read that failed). The
+ // reserver answers unreadable and reclaims nothing.
+ reclaimUnreadable
+ // reclaimYes: a clean negative after the fence. The entries go.
+ reclaimYes
+)
+
// offlineReclaimable is the reclaim rule for one operation's entries, all
// older than offlineLockGrace (the caller checked). Positive evidence only:
-// any read failure, malformed body, refusal or doubt keeps the reservation.
+// any read failure, malformed body or fence failure keeps the reservation
+// as reclaimUnreadable, and evidence that it is live as reclaimHeld.
//
// A move: (a) the holder's record is fenced (fenceRecord, under the
// holder's s.lockApp then withHostname, and admitted) and the fence answers
@@ -310,39 +344,46 @@ func (s *Server) offlineInTheWay(ctx context.Context, conflicts map[offlineOp]ma
// ETag), so a paused door that resumes after a reclaim cannot write.
//
// A cleanup: the owning box's status settles it (cleanupReportEvidence).
-func (s *Server) offlineReclaimable(ctx context.Context, entries map[string]moves.OfflineReservation) bool {
+//
+// No cleanup evidence yet (a nil cleanupReports) is held: an unsettled
+// cleanup is what every cleanup reservation is until a report says
+// otherwise.
+func (s *Server) offlineReclaimable(ctx context.Context, entries map[string]moves.OfflineReservation) reclaimVerdict {
for key, e := range entries {
switch e.Kind {
case moves.OfflineKindMove:
return s.offlineMoveReclaimable(ctx, e)
case moves.OfflineKindCleanup:
if s.cleanupReports == nil {
- return false
+ return reclaimHeld
}
ok, err := s.cleanupReports.CleanupSettled(ctx, key, e.ID)
if err != nil {
log.Printf("console: offline move: cleanup %s on %s: status: %v (kept)", e.ID, key, err)
- return false
+ return reclaimUnreadable
}
if !ok {
- return false
+ return reclaimHeld
}
default:
- return false
+ return reclaimUnreadable
}
}
- return len(entries) > 0
+ if len(entries) == 0 {
+ return reclaimHeld
+ }
+ return reclaimYes
}
-func (s *Server) offlineMoveReclaimable(ctx context.Context, e moves.OfflineReservation) bool {
+func (s *Server) offlineMoveReclaimable(ctx context.Context, e moves.OfflineReservation) reclaimVerdict {
carries := func(rec *provision.Record) bool { return rec != nil && rec.Move != nil && rec.Move.ID == e.ID }
pre := s.observeRecord(ctx, e.Account, e.App, e.Host)
switch {
case pre.Verdict == provision.LifetimeUnreadable, pre.Verdict == provision.LifetimeCorrupt:
log.Printf("console: offline move: reservation %s of %s: the record could not be read (kept): %v", e.ID, e.Host, pre.Err)
- return false
+ return reclaimUnreadable
case pre.Present && carries(pre.Rec):
- return false
+ return reclaimHeld
}
// LOCK ORDER: the holder's app lock, then its hostname exclusion (the
// lock table in lifetimes.go), held around the fence only. reserveBoxes'
@@ -359,32 +400,43 @@ func (s *Server) offlineMoveReclaimable(ctx context.Context, e moves.OfflineRese
}
res, obs, ferr = s.fenceRecord(ctx, e.Account, e.App, e.Host, fenceOwner{})
}()
- if ferr != nil {
+ switch {
+ case errors.Is(ferr, errLifetimeSettling):
+ // Another lifetime operation of the host is pending: it is live
+ // evidence, and the fence waits for it.
+ return reclaimHeld
+ case ferr != nil:
log.Printf("console: offline move: reservation %s of %s: not fenced (kept): %v", e.ID, e.Host, ferr)
- return false
+ return reclaimUnreadable
}
switch {
case res == fenceAbsent:
- case res == fenced && obs.Rec != nil && obs.Verdict != provision.LifetimeCorrupt && !carries(obs.Rec):
+ case res == fenced && (obs.Rec == nil || obs.Verdict == provision.LifetimeCorrupt):
+ return reclaimUnreadable
+ case res == fenced && !carries(obs.Rec):
default:
- return false
+ return reclaimHeld
}
if s.selfServeRows == nil {
- return true
+ return reclaimYes
}
done, err := s.selfServeRows.MoveTerminalOrAbsent(e.ID)
- if err != nil || !done {
- if err != nil {
- log.Printf("console: offline move: reservation %s: its row: %v (kept)", e.ID, err)
- }
- return false
+ if err != nil {
+ log.Printf("console: offline move: reservation %s: its row: %v (kept)", e.ID, err)
+ return reclaimUnreadable
+ }
+ if !done {
+ return reclaimHeld
}
issued, err := s.selfServeRows.HostHasIssuedRequest(e.Account, e.App, e.Host)
if err != nil {
log.Printf("console: offline move: reservation %s: the rows of %s: %v (kept)", e.ID, e.Host, err)
- return false
+ return reclaimUnreadable
+ }
+ if issued {
+ return reclaimHeld
}
- return !issued
+ return reclaimYes
}
// releaseBoxes removes every entry of the (kind, id) operation and rewrites