amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package lease_test
2
3import (
4 "bytes"
5 "context"
6 "encoding/json"
7 "errors"
8 "net/http"
9 "net/http/httptest"
10 "strings"
11 "sync"
12 "testing"
13 "time"
14
15 "github.com/carlosframework/platform/internal/lease"
16 "github.com/carlosframework/platform/internal/lease/leasetest"
17)
18
19const (
20 tombLoc = "instances/h.example.com"
21 tombMove = "moved:mv_01AAAAAAAAAAAAAAAAAAAAAAAA"
22 tombOther = "moved:mv_01BBBBBBBBBBBBBBBBBBBBBBBB"
23 tombRegion = "test-region"
24)
25
26// tombStore is leasetest's S3 server behind a wrapper that runs a hook in the
27// window between the code under test's read of lock.json and its conditional
28// write, so a race is a request ordering, not a sleep. The conditions
29// themselves (If-Match, If-None-Match) are leasetest's own.
30type tombStore struct {
31 *leasetest.VersionedBucket
32 mu sync.Mutex
33 before func(method string)
34 writes []string // "PUT" and "DELETE" requests that reached lock.json, in order
35}
36
37// beforeWrite arms hook for the next write to lock.json, once.
38func (s *tombStore) beforeWrite(hook func(method string)) {
39 s.mu.Lock()
40 defer s.mu.Unlock()
41 s.before = hook
42}
43
44func tombFixture(t *testing.T) *tombStore {
45 t.Helper()
46 bucket := leasetest.NewVersionedBucket()
47 bucket.AddBucket("data")
48 inner, _ := leasetest.NewS3Server(bucket)
49 s := &tombStore{VersionedBucket: bucket}
50 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
51 key := strings.TrimPrefix(r.URL.Path, "/data/")
52 if strings.HasSuffix(key, "lock.json") && (r.Method == http.MethodPut || r.Method == http.MethodDelete) {
53 s.mu.Lock()
54 hook := s.before
55 s.before = nil
56 s.writes = append(s.writes, r.Method)
57 s.mu.Unlock()
58 if hook != nil {
59 hook(r.Method)
60 }
61 }
62 inner.ServeHTTP(w, r)
63 }))
64 t.Cleanup(srv.Close)
65 t.Setenv("AWS_ACCESS_KEY_ID", "test-key")
66 t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
67 t.Setenv("AWS_SESSION_TOKEN", "")
68 t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
69 t.Setenv("AWS_CONFIG_FILE", t.TempDir()+"/absent")
70 t.Setenv("AWS_SHARED_CREDENTIALS_FILE", t.TempDir()+"/absent")
71 t.Setenv("CARLOS_S3_ENDPOINT", srv.URL)
72 return s
73}
74
75func (s *tombStore) methods() []string {
76 s.mu.Lock()
77 defer s.mu.Unlock()
78 return append([]string(nil), s.writes...)
79}
80
81func (s *tombStore) writeCount() int {
82 s.mu.Lock()
83 defer s.mu.Unlock()
84 return len(s.writes)
85}
86
87type tombLock struct {
88 Generation int64 `json:"generation"`
89 ExpiresAt time.Time `json:"expires_at"`
90 Owner string `json:"owner"`
91}
92
93func (s *tombStore) putLock(t *testing.T, loc, owner string, gen int64, expires time.Time) {
94 t.Helper()
95 body, err := json.Marshal(tombLock{Generation: gen, ExpiresAt: expires.UTC(), Owner: owner})
96 if err != nil {
97 t.Fatal(err)
98 }
99 s.Put("data", loc+"/lock.json", body)
100}
101
102func (s *tombStore) lock(t *testing.T, loc string) (tombLock, []byte, bool) {
103 t.Helper()
104 body, ok := s.Current("data", loc+"/lock.json")
105 if !ok {
106 return tombLock{}, nil, false
107 }
108 var l tombLock
109 if err := json.Unmarshal(body, &l); err != nil {
110 t.Fatalf("lock.json is not a lock: %v", err)
111 }
112 return l, body, true
113}
114
115func wantHeld(t *testing.T, err error, owner string) {
116 t.Helper()
117 var held *lease.HeldError
118 if !errors.As(err, &held) {
119 t.Fatalf("err = %v, want a *HeldError", err)
120 }
121 if held.Owner != owner {
122 t.Fatalf("held by %q, want %q", held.Owner, owner)
123 }
124}
125
126func near(t *testing.T, got time.Time, want time.Duration) {
127 t.Helper()
128 if d := time.Until(got) - want; d < -time.Minute || d > time.Minute {
129 t.Fatalf("expiry %s is %s from now, want about %s", got, time.Until(got), want)
130 }
131}
132
133func isMine(owner string) func(string) bool {
134 return func(o string) bool { return o == owner }
135}
136
137// A tombstone is a lease like any other: Acquire on the host's location
138// fails with a HeldError that names the move.
139func TestTombstoneBlocksAcquire(t *testing.T) {
140 s := tombFixture(t)
141 ctx := context.Background()
142 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, 2*time.Hour); err != nil {
143 t.Fatal(err)
144 }
145 l, _, ok := s.lock(t, tombLoc)
146 if !ok || l.Owner != tombMove {
147 t.Fatalf("lock = %+v present %v, want owner %s", l, ok, tombMove)
148 }
149 near(t, l.ExpiresAt, 2*time.Hour)
150 _, err := lease.Acquire(ctx, "data", tombLoc, tombRegion)
151 wantHeld(t, err, tombMove)
152 var held *lease.HeldError
153 errors.As(err, &held)
154 near(t, held.ExpiresAt, 2*time.Hour)
155 // The trailing-slash form ReplicaSource hands over reads the same lock.
156 got, owner, _, err := lease.LockState(ctx, "data", tombLoc+"/", tombRegion)
157 if err != nil || !got || owner != tombMove {
158 t.Fatalf("LockState = held %v owner %q err %v", got, owner, err)
159 }
160}
161
162func TestTombstoneNeverReplacesALiveHolder(t *testing.T) {
163 s := tombFixture(t)
164 ctx := context.Background()
165 held, err := lease.Acquire(ctx, "data", tombLoc, tombRegion)
166 if err != nil {
167 t.Fatal(err)
168 }
169 t.Cleanup(func() { _ = held.Release(context.Background()) })
170 _, before, _ := s.lock(t, tombLoc)
171 writes := s.writeCount()
172 err = lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour)
173 var he *lease.HeldError
174 if !errors.As(err, &he) || strings.HasPrefix(he.Owner, "moved:") {
175 t.Fatalf("Tombstone over a live holder = %v, want a *HeldError naming the holder", err)
176 }
177 if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
178 t.Fatalf("lock.json changed: %s -> %s", before, after)
179 }
180 if s.writeCount() != writes {
181 t.Fatal("Tombstone wrote to a live holder's lock")
182 }
183 if err := held.Err(); err != nil {
184 t.Fatalf("the holder lost its lease: %v", err)
185 }
186 // Reassert is no way round it either.
187 err = lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour)
188 if !errors.As(err, &he) {
189 t.Fatalf("Reassert over a live holder = %v, want a *HeldError", err)
190 }
191 if s.writeCount() != writes {
192 t.Fatal("Reassert wrote to a live holder's lock")
193 }
194}
195
196// Free means what Acquire means by it: the expiry has passed. There is no
197// RenewMargin grace, so a lease that lapsed a moment ago is taken by Tombstone
198// exactly as lease.Acquire would take it. The two are run side by side on
199// identical locks so they cannot drift.
200func TestTombstoneAndAcquireAgreeOnFree(t *testing.T) {
201 ctx := context.Background()
202 for name, tc := range map[string]struct {
203 expires time.Time
204 free bool
205 }{
206 "expired a second ago": {time.Now().Add(-time.Second), true},
207 "expired within the margin": {time.Now().Add(-lease.RenewMargin / 2), true},
208 "expires in a minute": {time.Now().Add(time.Minute), false},
209 } {
210 t.Run(name, func(t *testing.T) {
211 s := tombFixture(t)
212 s.putLock(t, tombLoc, "box-a:1", 3, tc.expires)
213 _, before, _ := s.lock(t, tombLoc)
214 acq, acqErr := lease.Acquire(ctx, "data", tombLoc, tombRegion)
215 if acqErr == nil {
216 _ = acq.Release(ctx)
217 }
218 // Put the same lock back for Tombstone.
219 s.putLock(t, tombLoc, "box-a:1", 3, tc.expires)
220 tombErr := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour)
221 if (acqErr == nil) != tc.free || (tombErr == nil) != tc.free {
222 t.Fatalf("free %v: Acquire err %v, Tombstone err %v", tc.free, acqErr, tombErr)
223 }
224 if !tc.free {
225 wantHeld(t, tombErr, "box-a:1")
226 if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
227 t.Fatal("a live lease was replaced")
228 }
229 }
230 })
231 }
232}
233
234// An own tombstone that has expired is free like any other: Tombstone writes
235// a fresh one rather than reporting the dead one held.
236func TestTombstoneRewritesItsOwnExpiredTombstone(t *testing.T) {
237 s := tombFixture(t)
238 s.putLock(t, tombLoc, tombMove, 4, time.Now().Add(-time.Second))
239 if err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
240 t.Fatal(err)
241 }
242 l, _, _ := s.lock(t, tombLoc)
243 if l.Owner != tombMove || l.Generation != 5 {
244 t.Fatalf("lock = %+v, want owner %s at generation 5", l, tombMove)
245 }
246 near(t, l.ExpiresAt, time.Hour)
247}
248
249// The write loses its condition and AcquireLease cannot read a winner back:
250// there is no holder to name, so the answer is ErrTombstoneChanged, never a
251// HeldError with an empty owner.
252func TestTombstoneWriteLostThenReadLostIsChanged(t *testing.T) {
253 s := tombFixture(t)
254 s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour))
255 s.beforeWrite(func(string) { s.BareDelete("data", tombLoc+"/lock.json") })
256 err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour)
257 var he *lease.HeldError
258 if !errors.Is(err, lease.ErrTombstoneChanged) || errors.As(err, &he) {
259 t.Fatalf("err = %v, want ErrTombstoneChanged and no HeldError", err)
260 }
261 if _, _, ok := s.lock(t, tombLoc); ok {
262 t.Fatal("Tombstone wrote over the vanished lock")
263 }
264}
265
266// The write loses its condition and the read-back shows this very owner
267// (another pass of the same move got there first): nothing to do.
268func TestTombstoneWriteLostToItsOwnOwnerIsNil(t *testing.T) {
269 s := tombFixture(t)
270 s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour))
271 var theirs []byte
272 s.beforeWrite(func(string) {
273 s.putLock(t, tombLoc, tombMove, 4, time.Now().Add(2*time.Hour))
274 _, theirs, _ = s.lock(t, tombLoc)
275 })
276 if err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
277 t.Fatalf("err = %v, want nil", err)
278 }
279 if _, now, _ := s.lock(t, tombLoc); string(now) != string(theirs) {
280 t.Fatalf("the winner's lock was overwritten: %s -> %s", theirs, now)
281 }
282}
283
284func TestTombstoneReplacesAnExpiredLease(t *testing.T) {
285 s := tombFixture(t)
286 s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour))
287 if err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
288 t.Fatal(err)
289 }
290 l, _, _ := s.lock(t, tombLoc)
291 if l.Owner != tombMove || l.Generation != 4 {
292 t.Fatalf("lock = %+v, want owner %s at generation 4", l, tombMove)
293 }
294 near(t, l.ExpiresAt, time.Hour)
295}
296
297func TestTombstoneByTheSameOwnerIsANoOp(t *testing.T) {
298 s := tombFixture(t)
299 ctx := context.Background()
300 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
301 t.Fatal(err)
302 }
303 _, first, _ := s.lock(t, tombLoc)
304 writes := s.writeCount()
305 // A longer ttl on the repeat changes nothing: Reassert is the call that moves it.
306 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, 48*time.Hour); err != nil {
307 t.Fatal(err)
308 }
309 if _, second, _ := s.lock(t, tombLoc); string(second) != string(first) || s.writeCount() != writes {
310 t.Fatalf("the repeat wrote: %s -> %s", first, second)
311 }
312}
313
314func TestTombstoneRefusesAnotherMove(t *testing.T) {
315 s := tombFixture(t)
316 ctx := context.Background()
317 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
318 t.Fatal(err)
319 }
320 _, before, _ := s.lock(t, tombLoc)
321 wantHeld(t, lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombOther, time.Hour), tombMove)
322 wantHeld(t, lease.Reassert(ctx, "data", tombLoc, tombRegion, tombOther, time.Hour), tombMove)
323 if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
324 t.Fatalf("another move's tombstone was overwritten: %s -> %s", before, after)
325 }
326}
327
328func TestTombstoneRefusesWhatItShouldNotWrite(t *testing.T) {
329 s := tombFixture(t)
330 ctx := context.Background()
331 for _, owner := range []string{"", "box-a:1", "moved:", "move:mv_1", "Moved:mv_1"} {
332 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, owner, time.Hour); !errors.Is(err, lease.ErrTombstoneOwner) {
333 t.Fatalf("Tombstone owner %q = %v, want ErrTombstoneOwner", owner, err)
334 }
335 if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, owner, time.Hour); !errors.Is(err, lease.ErrTombstoneOwner) {
336 t.Fatalf("Reassert owner %q = %v, want ErrTombstoneOwner", owner, err)
337 }
338 }
339 for _, ttl := range []time.Duration{0, -time.Hour, lease.MaxTombstoneTTL + time.Second, 30 * 24 * time.Hour} {
340 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, ttl); !errors.Is(err, lease.ErrTombstoneTTL) {
341 t.Fatalf("Tombstone ttl %s = %v, want ErrTombstoneTTL", ttl, err)
342 }
343 if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, ttl); !errors.Is(err, lease.ErrTombstoneTTL) {
344 t.Fatalf("Reassert ttl %s = %v, want ErrTombstoneTTL", ttl, err)
345 }
346 }
347 for _, loc := range []string{"", "/abs", "../x", "a/../b", `a\b`} {
348 if err := lease.Tombstone(ctx, "data", loc, tombRegion, tombMove, time.Hour); !errors.Is(err, lease.ErrLockLocation) {
349 t.Fatalf("Tombstone location %q = %v, want ErrLockLocation", loc, err)
350 }
351 }
352 if s.writeCount() != 0 {
353 t.Fatal("a refused call wrote")
354 }
355 // Seven days exactly is the longest there is.
356 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, lease.MaxTombstoneTTL); err != nil {
357 t.Fatal(err)
358 }
359 l, _, _ := s.lock(t, tombLoc)
360 near(t, l.ExpiresAt, lease.MaxTombstoneTTL)
361 if time.Until(l.ExpiresAt) > lease.MaxTombstoneTTL {
362 t.Fatalf("expiry %s is beyond 7 days", l.ExpiresAt)
363 }
364}
365
366func TestReassertMovesTheExpiry(t *testing.T) {
367 s := tombFixture(t)
368 ctx := context.Background()
369 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
370 t.Fatal(err)
371 }
372 first, _, _ := s.lock(t, tombLoc)
373 if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, 48*time.Hour); err != nil {
374 t.Fatal(err)
375 }
376 second, _, _ := s.lock(t, tombLoc)
377 if second.Owner != tombMove || second.Generation != first.Generation {
378 t.Fatalf("lock = %+v, want the same owner and generation as %+v", second, first)
379 }
380 near(t, second.ExpiresAt, 48*time.Hour)
381 // And it can be pulled in again: the expiry is what the caller says.
382 if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
383 t.Fatal(err)
384 }
385 third, _, _ := s.lock(t, tombLoc)
386 near(t, third.ExpiresAt, time.Hour)
387 _, err := lease.Acquire(ctx, "data", tombLoc, tombRegion)
388 wantHeld(t, err, tombMove)
389}
390
391// With nothing live to keep, Reassert reports the stop lost and writes
392// nothing: absent, its own tombstone expired or lifted, and someone else's
393// expired lease. It never takes a tombstone again (a wake may have used the
394// lease in between).
395func TestReassertWithNothingToReassertReportsTheStopLost(t *testing.T) {
396 s := tombFixture(t)
397 ctx := context.Background()
398 if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); !errors.Is(err, lease.ErrTombstoneLost) {
399 t.Fatalf("absent: %v, want ErrTombstoneLost", err)
400 }
401 if _, _, ok := s.lock(t, tombLoc); ok || s.writeCount() != 0 {
402 t.Fatalf("absent: a lock was written (%d writes)", s.writeCount())
403 }
404 for _, c := range []struct {
405 name string
406 owner string
407 expires time.Time
408 }{
409 {"its own, expired", tombMove, time.Now().Add(-time.Hour)},
410 {"its own, lifted", tombMove, time.Unix(0, 0)},
411 {"someone else's, expired", tombOther, time.Now().Add(-time.Hour)},
412 {"an ordinary lease, expired", "box-a:1", time.Now().Add(-time.Second)},
413 } {
414 s.putLock(t, tombLoc, c.owner, 5, c.expires)
415 _, before, _ := s.lock(t, tombLoc)
416 writes := s.writeCount()
417 if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); !errors.Is(err, lease.ErrTombstoneLost) {
418 t.Fatalf("%s: %v, want ErrTombstoneLost", c.name, err)
419 }
420 if _, after, _ := s.lock(t, tombLoc); !bytes.Equal(before, after) || s.writeCount() != writes {
421 t.Fatalf("%s: the lock was rewritten", c.name)
422 }
423 }
424}
425
426func TestReassertRefusesAHolderThatIsNotTheMove(t *testing.T) {
427 s := tombFixture(t)
428 s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(time.Hour))
429 _, before, _ := s.lock(t, tombLoc)
430 wantHeld(t, lease.Reassert(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour), "box-a:1")
431 if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
432 t.Fatal("Reassert overwrote a live holder")
433 }
434}
435
436// The lock is rewritten between Reassert's read and its write: the write
437// carries the stale ETag, S3 refuses it, and the rewrite stands.
438func TestReassertFailsWhenTheLockChangesUnderIt(t *testing.T) {
439 for name, rewrite := range map[string]func(s *tombStore, t *testing.T){
440 "another move takes it": func(s *tombStore, t *testing.T) {
441 s.putLock(t, tombLoc, tombOther, 9, time.Now().Add(3*time.Hour))
442 },
443 "the same move rewrites it": func(s *tombStore, t *testing.T) {
444 s.putLock(t, tombLoc, tombMove, 9, time.Now().Add(3*time.Hour))
445 },
446 "it is removed": func(s *tombStore, t *testing.T) {
447 s.BareDelete("data", tombLoc+"/lock.json")
448 },
449 } {
450 t.Run(name, func(t *testing.T) {
451 s := tombFixture(t)
452 ctx := context.Background()
453 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
454 t.Fatal(err)
455 }
456 var after []byte
457 var present bool
458 s.beforeWrite(func(method string) {
459 if method != http.MethodPut {
460 t.Errorf("Reassert sent a %s", method)
461 }
462 rewrite(s, t)
463 _, after, present = s.lock(t, tombLoc)
464 })
465 err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, 48*time.Hour)
466 if !errors.Is(err, lease.ErrTombstoneChanged) {
467 t.Fatalf("err = %v, want ErrTombstoneChanged", err)
468 }
469 _, now, ok := s.lock(t, tombLoc)
470 if ok != present || string(now) != string(after) {
471 t.Fatalf("Reassert overwrote the rewrite: %s -> %s", after, now)
472 }
473 })
474 }
475}
476
477// Tombstone's own windows: the lock is absent or expired when it reads, and a
478// live holder is there when it writes.
479func TestTombstoneLosesToAHolderThatArrivesFirst(t *testing.T) {
480 for name, setup := range map[string]func(s *tombStore, t *testing.T){
481 "absent": func(s *tombStore, t *testing.T) {},
482 "expired": func(s *tombStore, t *testing.T) { s.putLock(t, tombLoc, "box-a:1", 3, time.Now().Add(-time.Hour)) },
483 } {
484 t.Run(name, func(t *testing.T) {
485 s := tombFixture(t)
486 setup(s, t)
487 var holder []byte
488 s.beforeWrite(func(string) {
489 s.putLock(t, tombLoc, "box-b:2", 7, time.Now().Add(15*time.Minute))
490 _, holder, _ = s.lock(t, tombLoc)
491 })
492 err := lease.Tombstone(context.Background(), "data", tombLoc, tombRegion, tombMove, time.Hour)
493 wantHeld(t, err, "box-b:2")
494 if _, now, _ := s.lock(t, tombLoc); string(now) != string(holder) {
495 t.Fatalf("Tombstone replaced the holder that arrived first: %s -> %s", holder, now)
496 }
497 })
498 }
499}
500
501func TestLiftTombstone(t *testing.T) {
502 s := tombFixture(t)
503 ctx := context.Background()
504 called := ""
505 mine := func(o string) bool { called = o; return o == tombMove }
506
507 // Absent: nothing to lift, and mine is not consulted.
508 if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, mine); lifted || err != nil || called != "" {
509 t.Fatalf("absent: lifted %v err %v consulted %q", lifted, err, called)
510 }
511
512 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
513 t.Fatal(err)
514 }
515 live, before, _ := s.lock(t, tombLoc)
516
517 // Another move's tombstone is not ours to lift.
518 lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombOther))
519 wantHeld(t, err, tombMove)
520 if lifted {
521 t.Fatal("lifted another move's tombstone")
522 }
523 if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
524 t.Fatal("the tombstone changed")
525 }
526 // No ownership test at all lifts nothing.
527 if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, nil); lifted || err == nil {
528 t.Fatalf("nil mine: lifted %v err %v", lifted, err)
529 }
530
531 // Ours lifts: by a conditional PUT, leaving the same owner and generation
532 // with a long-past expiry, never by a DELETE.
533 writes := s.writeCount()
534 lifted, err = lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, mine)
535 if err != nil || !lifted || called != tombMove {
536 t.Fatalf("lift: lifted %v err %v consulted %q", lifted, err, called)
537 }
538 if got := s.methods()[writes:]; len(got) != 1 || got[0] != http.MethodPut {
539 t.Fatalf("lift wrote %v, want one PUT", got)
540 }
541 l, _, ok := s.lock(t, tombLoc)
542 if !ok || l.Owner != tombMove || l.Generation != live.Generation || !l.ExpiresAt.Before(time.Now().Add(-lease.RenewMargin)) {
543 t.Fatalf("after the lift: %+v present %v, want %s at generation %d, long expired", l, ok, tombMove, live.Generation)
544 }
545 if held, owner, expires, err := lease.LockState(ctx, "data", tombLoc, tombRegion); err != nil || held || !lease.IsLiftedTombstone(owner, expires) {
546 t.Fatalf("LockState after the lift: held %v owner %q expires %v err %v", held, owner, expires, err)
547 }
548 // The host can be acquired again, and so can a new tombstone.
549 held, err := lease.Acquire(ctx, "data", tombLoc, tombRegion)
550 if err != nil {
551 t.Fatalf("Acquire after the lift: %v", err)
552 }
553 if err := held.Release(ctx); err != nil {
554 t.Fatal(err)
555 }
556}
557
558func TestLiftedTombstoneIsFreeToTheNextMove(t *testing.T) {
559 s := tombFixture(t)
560 ctx := context.Background()
561 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
562 t.Fatal(err)
563 }
564 if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); err != nil || !lifted {
565 t.Fatalf("lifted %v err %v", lifted, err)
566 }
567 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombOther, time.Hour); err != nil {
568 t.Fatalf("Tombstone over a lifted one: %v", err)
569 }
570 if l, _, _ := s.lock(t, tombLoc); l.Owner != tombOther {
571 t.Fatalf("lock = %+v", l)
572 }
573}
574
575// Already lifted, however it came about, is (false, nil) with nothing written.
576func TestLiftTombstoneAlreadyLifted(t *testing.T) {
577 s := tombFixture(t)
578 ctx := context.Background()
579 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
580 t.Fatal(err)
581 }
582 if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); err != nil || !lifted {
583 t.Fatalf("first lift: %v %v", lifted, err)
584 }
585 writes := s.writeCount()
586 for i := 0; i < 2; i++ {
587 if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil {
588 t.Fatalf("already lifted: lifted %v err %v", lifted, err)
589 }
590 }
591 // An own tombstone that simply ran out is lifted too.
592 s.putLock(t, tombLoc, tombMove, 4, time.Now().Add(-48*time.Hour))
593 writes = s.writeCount()
594 if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil {
595 t.Fatalf("expired own: lifted %v err %v", lifted, err)
596 }
597 if s.writeCount() != writes {
598 t.Fatal("lifting an already-lifted tombstone wrote")
599 }
600 // And so is an absent one.
601 s.BareDelete("data", tombLoc+"/lock.json")
602 if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil {
603 t.Fatalf("absent: lifted %v err %v", lifted, err)
604 }
605}
606
607func TestIsLiftedTombstone(t *testing.T) {
608 past, future := time.Now().Add(-time.Second), time.Now().Add(time.Hour)
609 for name, tc := range map[string]struct {
610 owner string
611 expires time.Time
612 want bool
613 }{
614 "expired move": {tombMove, past, true},
615 "long expired move": {tombMove, time.Unix(0, 0), true},
616 "live move": {tombMove, future, false},
617 "expired ordinary lease": {"box-a:1", past, false},
618 "live ordinary lease": {"box-a:1", future, false},
619 "absent (zero owner/exp)": {"", time.Time{}, false},
620 "bare prefix": {"moved:", past, false},
621 "zero expiry": {tombMove, time.Time{}, false},
622 } {
623 if got := lease.IsLiftedTombstone(tc.owner, tc.expires); got != tc.want {
624 t.Errorf("%s: IsLiftedTombstone(%q, %v) = %v, want %v", name, tc.owner, tc.expires, got, tc.want)
625 }
626 }
627}
628
629// Whatever mine says, only a moved:<id> lease is lifted: a live process's
630// lease is never this function's to touch.
631func TestLiftTombstoneLeavesOrdinaryLeasesAlone(t *testing.T) {
632 s := tombFixture(t)
633 ctx := context.Background()
634 for name, expires := range map[string]time.Time{"live": time.Now().Add(time.Hour), "expired": time.Now().Add(-time.Hour)} {
635 s.putLock(t, tombLoc, "box-a:1", 3, expires)
636 _, before, _ := s.lock(t, tombLoc)
637 lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, func(string) bool { return true })
638 wantHeld(t, err, "box-a:1")
639 if lifted {
640 t.Fatalf("%s: lifted an ordinary lease", name)
641 }
642 if _, after, _ := s.lock(t, tombLoc); string(after) != string(before) {
643 t.Fatalf("%s: the lease changed", name)
644 }
645 }
646 if s.writeCount() != 0 { // putLock writes the bucket directly; only requests count
647 t.Fatal("a refused lift wrote")
648 }
649}
650
651func TestLiftTombstoneFailsWhenTheLockChangesUnderIt(t *testing.T) {
652 ctx := context.Background()
653 t.Run("rewritten", func(t *testing.T) {
654 s := tombFixture(t)
655 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
656 t.Fatal(err)
657 }
658 var after []byte
659 s.beforeWrite(func(method string) {
660 if method != http.MethodPut {
661 t.Errorf("LiftTombstone sent a %s", method)
662 }
663 // Still ours by owner, but a different lock: Reassert beat the lift.
664 s.putLock(t, tombLoc, tombMove, 8, time.Now().Add(72*time.Hour))
665 _, after, _ = s.lock(t, tombLoc)
666 })
667 lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove))
668 if lifted || !errors.Is(err, lease.ErrTombstoneChanged) {
669 t.Fatalf("lifted %v err %v, want (false, ErrTombstoneChanged)", lifted, err)
670 }
671 if _, now, ok := s.lock(t, tombLoc); !ok || string(now) != string(after) {
672 t.Fatalf("the lift overwrote the rewrite: %s", now)
673 }
674 })
675 t.Run("taken over by a holder", func(t *testing.T) {
676 s := tombFixture(t)
677 // Live tombstone read; before the write a holder cannot legitimately
678 // appear, but a rewrite by anything must still stand.
679 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
680 t.Fatal(err)
681 }
682 var after []byte
683 s.beforeWrite(func(string) {
684 s.putLock(t, tombLoc, "box-b:2", 5, time.Now().Add(15*time.Minute))
685 _, after, _ = s.lock(t, tombLoc)
686 })
687 lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove))
688 if lifted || !errors.Is(err, lease.ErrTombstoneChanged) {
689 t.Fatalf("lifted %v err %v, want (false, ErrTombstoneChanged)", lifted, err)
690 }
691 if _, now, _ := s.lock(t, tombLoc); string(now) != string(after) {
692 t.Fatalf("the lift overwrote a live holder's lock: %s", now)
693 }
694 })
695 t.Run("removed", func(t *testing.T) {
696 s := tombFixture(t)
697 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
698 t.Fatal(err)
699 }
700 s.beforeWrite(func(string) { s.BareDelete("data", tombLoc+"/lock.json") })
701 if lifted, err := lease.LiftTombstone(ctx, "data", tombLoc, tombRegion, isMine(tombMove)); lifted || err != nil {
702 t.Fatalf("lifted %v err %v, want (false, nil): already gone", lifted, err)
703 }
704 if _, _, ok := s.lock(t, tombLoc); ok {
705 t.Fatal("the lift re-created a removed lock")
706 }
707 })
708}
709
710// The tombstone is a lock.json like any other, and MirrorPrefix, which copies
711// a host's replica to a staging prefix, leaves lock.json alone both ways.
712func TestMirrorPrefixStillSkipsTheTombstone(t *testing.T) {
713 s := tombFixture(t)
714 ctx := context.Background()
715 s.Put("data", "instances/h.example.com/generations/a/snapshot", []byte("snap"))
716 if err := lease.Tombstone(ctx, "data", tombLoc, tombRegion, tombMove, time.Hour); err != nil {
717 t.Fatal(err)
718 }
719 // A staging lease of the target's own, under the destination prefix.
720 s.putLock(t, "moves/h.example.com", "box-z:9", 1, time.Now().Add(time.Hour))
721 _, staged, _ := s.lock(t, "moves/h.example.com")
722
723 m := bucketMirror{s.VersionedBucket}
724 r, err := lease.MirrorPrefix(ctx, m, "data", tombLoc+"/", "data", "moves/h.example.com/", lease.Receipts{})
725 if err != nil || r.Changed != 1 {
726 t.Fatalf("round = %+v, %v; want only the snapshot to move", r, err)
727 }
728 if _, now, _ := s.lock(t, "moves/h.example.com"); string(now) != string(staged) {
729 t.Fatal("the staging lease was touched")
730 }
731 if l, _, ok := s.lock(t, tombLoc); !ok || l.Owner != tombMove {
732 t.Fatalf("the tombstone was touched: %+v", l)
733 }
734 // Reasserting the tombstone is churn the next round does not see.
735 if err := lease.Reassert(ctx, "data", tombLoc, tombRegion, tombMove, 24*time.Hour); err != nil {
736 t.Fatal(err)
737 }
738 if r, err := lease.MirrorPrefix(ctx, m, "data", tombLoc+"/", "data", "moves/h.example.com/", lease.Receipts{}); err != nil || r.Changed != 0 {
739 t.Fatalf("caught-up round = %+v, %v; want 0 despite the reassert", r, err)
740 }
741}
742
743// bucketMirror is a MirrorStore over the versioned bucket the S3 server also
744// serves, so MirrorPrefix runs over the very lock.json objects the tombstone
745// calls wrote (the S3 server does not speak ListObjectsV2).
746type bucketMirror struct{ b *leasetest.VersionedBucket }
747
748func (m bucketMirror) List(ctx context.Context, bucket, prefix string) (map[string]lease.MirrorObject, error) {
749 vs, err := m.b.ListVersions(ctx, bucket, prefix)
750 if err != nil {
751 return nil, err
752 }
753 out := map[string]lease.MirrorObject{}
754 for _, v := range vs {
755 if v.IsLatest && !v.DeleteMarker {
756 out[v.Key] = lease.MirrorObject{ETag: v.ETag, Size: v.Size}
757 }
758 }
759 return out, nil
760}
761
762func (m bucketMirror) Copy(ctx context.Context, srcBucket, srcKey, srcETag, dstBucket, dstKey string) (string, error) {
763 vs, err := m.b.ListVersions(ctx, srcBucket, srcKey)
764 if err != nil {
765 return "", err
766 }
767 for _, v := range vs {
768 if v.Key == srcKey && v.IsLatest && !v.DeleteMarker && v.ETag == srcETag {
769 _, etag, err := m.b.CopyVersion(ctx, srcBucket, srcKey, v.VersionID, dstBucket, dstKey, false)
770 return etag, err
771 }
772 }
773 return "", lease.ErrMirrorNotFound
774}
775
776func (m bucketMirror) Delete(_ context.Context, bucket, key string) error {
777 m.b.BareDelete(bucket, key)
778 return nil
779}
780
781func (m bucketMirror) Get(_ context.Context, bucket, key string) ([]byte, error) {
782 if body, ok := m.b.Current(bucket, key); ok {
783 return body, nil
784 }
785 return nil, lease.ErrMirrorNotFound
786}
787
788func (m bucketMirror) Put(_ context.Context, bucket, key string, body []byte) error {
789 m.b.Put(bucket, key, body)
790 return nil
791}
792