amadan
self-serve job: participant pinning and report reads are judged at the pass's clock (fix round 2, item 7)
Diagnosis of CI's TestClaimsOffInstanceIDIsPublished failure (d0227a68): the witness moves the boxes' clock and the console's clock on together by 61 s, but two of the job's freshness judgements read the wall clock, not the pass's: pinParticipants (offlineReportCurrent) and readOfflineEvidence (ValidateOfflineReportFor's future skew). Reached less than a second after the advance, as on a fast runner, the box reports stamped 61 s ahead were a minute in the future at wall time: "the box ie-box has not reported recently" every round. On a slower box enough wall time passed to hide it. In production both clocks are the wall clock (selfServeNow is time.Now() with no test clock), so no production answer changes; the two judgements now use selfServeNow like the job's others, and a door's read, outside the pass, still reads the wall clock. The witness advances two minutes, past OfflineRequestSkew, so it fails on every machine without the fix. RED (this box): "after 60 rounds, still not in staging" with "waiting: the box ie-box has not reported recently", then with pinning fixed "offline report: written 1m58s in the future"; GREEN with both.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2 files changed,
+18
−5
internal/console/apiinstancemove_offline.go+9 −2internal/e2e/selfserve/selfserve_operator_test.go+9 −3
diff --git a/internal/console/apiinstancemove_offline.go b/internal/console/apiinstancemove_offline.go| index e55ce4324..070b12896 100644 |
| --- a/internal/console/apiinstancemove_offline.go |
| +++ b/internal/console/apiinstancemove_offline.go |
| @@ -232,7 +232,10 @@ func (e offlineEvidence) byRole(role string) []offlineReport { |
| // a request-bound report's future skew), logged. m nil is the legacy |
| // validation alone. A listing or a read that fails is the error. |
| func (s *Server) readOfflineEvidence(ctx context.Context, host, id string, m *provision.MoveState) (offlineEvidence, error) { |
| - now := time.Now() |
| + // The reports are judged at the pass's clock when the job reads them |
| + // (selfServeNow), as pinning is; a door's read is the wall clock (fix |
| + // round 2, item 7). |
| + now := s.selfServeNow() |
| prefix := moves.OfflineReportPrefix(host, id) |
| if prefix == "" { |
| return offlineEvidence{}, nil |
| @@ -657,7 +660,11 @@ func (s *Server) pinParticipants(ctx context.Context, sel *offlineSelection, req |
| return ref |
| } |
| } |
| - now := time.Now() |
| + // The reports are judged at the pass's clock when the self-serve |
| + // job pins (selfServeNow: its tick's clock, which a deadline witness |
| + // moves with the boxes'), as the job's other freshness judgements are; |
| + // a door's request reads the wall clock (fix round 2, item 7). |
| + now := s.selfServeNow() |
| sel.Source = nil |
| for _, b := range sel.Fleet.boxesAt(sel.From) { |
| p, ref := s.pinParticipant(ctx, sel.Fleet, b, s.homeRegionIn(b.Placement), requireIDs, id, now) |
diff --git a/internal/e2e/selfserve/selfserve_operator_test.go b/internal/e2e/selfserve/selfserve_operator_test.go| index f6c3aabd0..7bb2a2fdb 100644 |
| --- a/internal/e2e/selfserve/selfserve_operator_test.go |
| +++ b/internal/e2e/selfserve/selfserve_operator_test.go |
| @@ -325,12 +325,18 @@ func TestClaimsOffInstanceIDIsPublished(t *testing.T) { |
| t.Fatalf("the move pinned %+v", r.Participants) |
| } |
| // The moves worker asks again a minute on (the boxes' clock and the |
| - // console's moved together), and the source takes its role. |
| + // console's moved together), and the source takes its role. Two |
| + // minutes, so the boxes' stamps are past OfflineRequestSkew ahead of |
| + // the wall clock: every judgement the console's pass makes of them |
| + // must be at the pass's clock (moved with the boxes'), never the wall |
| + // clock (fix round 2, item 7: a fast runner reached staging's |
| + // participant pinning less than a second after a 61 s advance and |
| + // judged the reports a minute in the future). |
| if got := src.InstanceID(); got != "" { |
| t.Fatalf("the worker re-read the instance id inside its minute: %q", got) |
| } |
| - w.e.Clock().Advance(61 * time.Second) |
| - w.advance(61 * time.Second) |
| + w.e.Clock().Advance(2 * time.Minute) |
| + w.advance(2 * time.Minute) |
| if got := src.InstanceID(); got != "i-e2e-ie-box" { |
| t.Fatalf("the worker's instance id a minute on: %q", got) |
| } |