| index 033ceb635..4d0813f9a 100644 |
| --- a/internal/console/apiinstancemove_offline.go |
| +++ b/internal/console/apiinstancemove_offline.go |
| @@ -546,27 +546,57 @@ func (s *Server) handleAPIInstanceOfflineStart(w http.ResponseWriter, r *http.Re |
| writeRefusal(w, ref) |
| return |
| } |
| - got, ok := s.offlineWrite(w, r, d, start) |
| - if !ok { |
| - s.offlineReleaseLock(ctx, host, id) |
| + // The write is bounded well under offlineLockGrace, so the lock cannot be |
| + // judged stale while it is still in flight. |
| + wctx, cancel := context.WithTimeout(ctx, offlineStartWrite) |
| + defer cancel() |
| + got, err := s.offlineApply(wctx, d, start) |
| + if err != nil { |
| + // A failed write does not prove nothing was written (a commit whose |
| + // reply was lost). The lock is never released here: if the record |
| + // carries this move the start succeeded, and if it does not the |
| + // lock goes stale and frees itself. |
| + rctx, rcancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) |
| + defer rcancel() |
| + if cur, rerr := provision.GetRecord(rctx, s.cfg.Releases, d.acct.Sqid, d.app, host); rerr == nil && cur.Move.Offline() && cur.Move.ID == id { |
| + got, err = cur, nil |
| + } |
| + } |
| + if err != nil { |
| + s.offlineWriteFailed(w, d, err) |
| return |
| } |
| + s.bumpSerial(ctx) |
| writeJSON(w, http.StatusAccepted, s.offlineViewOf(ctx, got)) |
| } |
| |
| +// offlineStartWrite bounds the start door's record write. |
| +const offlineStartWrite = 30 * time.Second |
| + |
| +// offlineApply is one transition through the record writer. |
| +func (s *Server) offlineApply(ctx context.Context, d offlineDoor, mut func(*provision.Record) error) (*provision.Record, error) { |
| + return provision.SetRecordMove(ctx, s.cfg.Releases, d.acct.Sqid, d.app, d.host, mut) |
| +} |
| + |
| +// offlineWriteFailed answers a transition's failure: a refusal with its |
| +// recorded sentence, anything else as unwritten. |
| +func (s *Server) offlineWriteFailed(w http.ResponseWriter, d offlineDoor, err error) { |
| + if errors.Is(err, provision.ErrMoveConflict) { |
| + jsonError(w, http.StatusConflict, offlineConflict(err)) |
| + return |
| + } |
| + log.Printf("console: offline move %s/%s/%s: %v", d.acct.Sqid, d.app, d.host, err) |
| + jsonError(w, http.StatusBadGateway, "could not write the move. try again") |
| +} |
| + |
| // offlineWrite applies one transition and bumps the serial, or answers the |
| // refusal itself. When the move it leaves is cleared (finish, or the finish |
| // that ends an abort) it releases the lock. |
| func (s *Server) offlineWrite(w http.ResponseWriter, r *http.Request, d offlineDoor, mut func(*provision.Record) error) (*provision.Record, bool) { |
| ctx := r.Context() |
| - rec, err := provision.SetRecordMove(ctx, s.cfg.Releases, d.acct.Sqid, d.app, d.host, mut) |
| - if errors.Is(err, provision.ErrMoveConflict) { |
| - jsonError(w, http.StatusConflict, offlineConflict(err)) |
| - return nil, false |
| - } |
| + rec, err := s.offlineApply(ctx, d, mut) |
| if err != nil { |
| - log.Printf("console: offline move %s/%s/%s: %v", d.acct.Sqid, d.app, d.host, err) |
| - jsonError(w, http.StatusBadGateway, "could not write the move. try again") |
| + s.offlineWriteFailed(w, d, err) |
| return nil, false |
| } |
| s.bumpSerial(ctx) |
| @@ -795,7 +825,7 @@ func (s *Server) offlineSealCheck(f offlineFleet, ev offlineEvidence, mv *provis |
| // byte for byte. The seal is written once, so the hash holds for the whole |
| // move whatever the sealer's living report does each pass. |
| func offlineSealHolds(ev offlineEvidence, mv *provision.MoveState, host string) bool { |
| - return ev.seal != nil && mv.Source != nil && |
| + return ev.seal != nil && ev.seal.Box == mv.Sealer && mv.Source != nil && |
| offlineEvidenceMatches(mv.Source.Evidence, moves.OfflineSealKey(host, mv.ID), ev.seal.raw) |
| } |
| |
amadan