amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostops
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "os"
9 "reflect"
10 "testing"
11 "time"
12
13 "github.com/carlosframework/platform/internal/hostclaim"
14 "github.com/carlosframework/platform/internal/provision"
15 "github.com/carlosframework/platform/internal/release"
16)
17
18type authority struct {
19 err error
20 // selects is region HA's create-time choice; false is today's create.
21 // selected is the box it chooses (box-a when empty); executor is the
22 // box completion offers (box-a when empty); selections counts calls.
23 selects bool
24 selected, executor string
25 selections int
26}
27
28func (a *authority) SelectExecutor(context.Context, hostclaim.Observed, Record) (hostclaim.ExecutorSelection, bool, error) {
29 a.selections++
30 if !a.selects {
31 return hostclaim.ExecutorSelection{}, false, a.err
32 }
33 box := a.selected
34 if box == "" {
35 box = "box-a"
36 }
37 return hostclaim.ExecutorSelection{Box: box, Enrollment: "enrollment-" + box[len("box-"):]}, true, a.err
38}
39func (a *authority) Create(context.Context, hostclaim.Observed, Record, string) error { return a.err }
40func (a *authority) Executor(context.Context, hostclaim.Observed, Record) (string, error) {
41 if a.executor != "" {
42 return a.executor, a.err
43 }
44 return "box-a", a.err
45}
46func (a *authority) StartDelete(context.Context, hostclaim.Observed, Record) error { return a.err }
47func (a *authority) FirstPhase(context.Context, hostclaim.Observed) error { return a.err }
48func (a *authority) Inventory(context.Context, hostclaim.Observed) ([]string, []string, error) {
49 return []string{"ie", "us"}, []string{"box-a", "box-b"}, a.err
50}
51func (a *authority) Adoption(context.Context, Record) (string, string, []string, []string, error) {
52 return "ie", "box-a", []string{"us"}, []string{"box-b"}, a.err
53}
54func (a *authority) PurgeRetirement(context.Context, hostclaim.Observed) error { return a.err }
55
56type faultStore struct {
57 release.Store
58 failGet string
59 afterList func(string, []string) error
60 beforePut func(string) error
61 afterGet func(string)
62 afterPut func(string)
63 beforeDelete func(string, string) error
64 bareDeletes int
65}
66
67func (s *faultStore) List(ctx context.Context, prefix string) ([]string, error) {
68 keys, err := s.Store.List(ctx, prefix)
69 if err == nil && s.afterList != nil {
70 err = s.afterList(prefix, keys)
71 }
72 return keys, err
73}
74
75func (s *faultStore) Get(ctx context.Context, key string) ([]byte, string, error) {
76 if key == s.failGet {
77 return nil, "", errors.New("read failed")
78 }
79 b, tag, err := s.Store.Get(ctx, key)
80 if err == nil && s.afterGet != nil {
81 s.afterGet(key)
82 }
83 return b, tag, err
84}
85func (s *faultStore) Put(ctx context.Context, key string, b []byte, e string) (string, error) {
86 if s.beforePut != nil {
87 if err := s.beforePut(key); err != nil {
88 return "", err
89 }
90 }
91 tag, err := s.Store.Put(ctx, key, b, e)
92 if err == nil && s.afterPut != nil {
93 s.afterPut(key)
94 }
95 return tag, err
96}
97func (s *faultStore) Delete(context.Context, string) error {
98 s.bareDeletes++
99 return errors.New("bare delete forbidden")
100}
101func (s *faultStore) DeleteIfMatch(ctx context.Context, key, e string) error {
102 if s.beforeDelete != nil {
103 if err := s.beforeDelete(key, e); err != nil {
104 return err
105 }
106 }
107 return release.DeleteIfMatch(ctx, s.Store, key, e)
108}
109
110type fixture struct {
111 ctx context.Context
112 store *faultStore
113 c Controller
114 auth *authority
115 now time.Time
116 serial int
117}
118
119func newFixture(t *testing.T) *fixture {
120 t.Helper()
121 f := &fixture{ctx: context.Background(), now: time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC), auth: &authority{}}
122 f.store = &faultStore{Store: release.NewDirStore(t.TempDir())}
123 f.c = Controller{Store: f.store, Authority: f.auth, Now: func() time.Time { return f.now }, NewID: func() string { f.serial++; return fmt.Sprintf("id-%d", f.serial) }}
124 return f
125}
126func (f *fixture) record(t *testing.T, host string) Record {
127 t.Helper()
128 r, err := PrepareInstance(provision.Record{Schema: 1, Account: "bab", App: "jam", Host: host, Slug: "shop", Channel: "edge", Region: "ie"}, f.now, "ie")
129 if err != nil {
130 t.Fatal(err)
131 }
132 return r
133}
134func (f *fixture) serving(t *testing.T, host string) hostclaim.Observed {
135 t.Helper()
136 r := f.record(t, host)
137 before, err := hostclaim.Read(f.ctx, f.store, host)
138 if err != nil {
139 t.Fatal(err)
140 }
141 obs, err := f.c.Attempt(f.ctx, before, r, f.c.NewID(), "test")
142 if err != nil {
143 t.Fatal(err)
144 }
145 return obs
146}
147func TestAttemptUsesPreRegistrationObservation(t *testing.T) {
148 f := newFixture(t)
149 for _, host := range []string{"one.bab.example.com", "two.bab.example.com"} {
150 before, err := hostclaim.Read(f.ctx, f.store, host)
151 if err != nil {
152 t.Fatal(err)
153 }
154 fenced, err := hostclaim.Update(f.ctx, f.store, before, FencePurge("purge", f.now))
155 if err != nil {
156 t.Fatal(err)
157 }
158 if _, err = hostclaim.Update(f.ctx, f.store, fenced, ReleasePurgeFence("purge")); err != nil {
159 t.Fatal(err)
160 }
161 r := f.record(t, host)
162 if _, err = f.c.Attempt(f.ctx, before, r, "paused", "test"); !errors.Is(err, hostclaim.ErrConflict) {
163 t.Fatalf("late claim: %v", err)
164 }
165 if _, _, err = f.store.Get(f.ctx, r.Holder.Key); !errors.Is(err, os.ErrNotExist) {
166 t.Fatalf("record landed: %v", err)
167 }
168 }
169 if f.store.bareDeletes != 0 {
170 t.Fatal("deleted claim")
171 }
172}
173
174// TestCreateIsTodaysWithoutASelection pins dormancy (region HA §8 step 2):
175// with no create-time choice the claim carries no selected executor and
176// completion takes the box Executor names. With one, Attempt itself
177// records the selection in the create CAS (round 9 finding 2): selection
178// chooses box-b while completion offers box-a, so the completion refuses,
179// the persisted intent names box-b and its enrollment, and a completion
180// with box-b serves without selecting again.
181func TestCreateIsTodaysWithoutASelection(t *testing.T) {
182 f := newFixture(t)
183 obs := f.serving(t, "off.bab.example.com")
184 if obs.Claim.Op.Exec != "box-a" || obs.Claim.Op.Intent["selected_executor"] != "" || obs.Claim.Op.Intent["executor_enrollment"] != "" {
185 t.Fatalf("HA off: %+v", obs.Claim.Op)
186 }
187 f.auth.selects, f.auth.selected, f.auth.selections = true, "box-b", 0
188 const host = "on.bab.example.com"
189 before, err := hostclaim.Read(f.ctx, f.store, host)
190 if err != nil {
191 t.Fatal(err)
192 }
193 id := f.c.NewID()
194 if _, err := f.c.Attempt(f.ctx, before, f.record(t, host), id, "test"); !errors.Is(err, hostclaim.ErrWrongState) {
195 t.Fatalf("completion substituted box-a for the selected box-b: %v", err)
196 }
197 pending, err := hostclaim.Read(f.ctx, f.store, host)
198 if err != nil {
199 t.Fatal(err)
200 }
201 if pending.Claim.Op.Kind != hostclaim.KindCreating || pending.Claim.Op.Intent["selected_executor"] != "box-b" || pending.Claim.Op.Intent["executor_enrollment"] != "enrollment-b" {
202 t.Fatalf("HA on, the persisted intent: %+v", pending.Claim.Op)
203 }
204 f.auth.executor = "box-b"
205 done, err := f.c.Complete(f.ctx, host, id)
206 if err != nil || done.Claim.Op.Kind != hostclaim.KindServing || done.Claim.Op.Exec != "box-b" {
207 t.Fatalf("completion with the selected box: %+v %v", done.Claim, err)
208 }
209 if f.auth.selections != 1 {
210 t.Fatalf("selection ran %d times, want once", f.auth.selections)
211 }
212}
213
214func TestRecoverCreateAbsentAndLateOrphan(t *testing.T) {
215 f := newFixture(t)
216 host := "shop.bab.example.com"
217 r := f.record(t, host)
218 before, _ := hostclaim.Read(f.ctx, f.store, host)
219 if _, err := hostclaim.Update(f.ctx, f.store, before, hostclaim.BeginCreate(r.Holder, r.Body, "create", "test", f.now)); err != nil {
220 t.Fatal(err)
221 }
222 f.now = f.now.Add(5 * time.Minute)
223 if err := f.c.RecoverCreate(f.ctx, host); !errors.Is(err, ErrPending) {
224 t.Fatal(err)
225 }
226 f.now = f.now.Add(time.Nanosecond)
227 if err := f.c.RecoverCreate(f.ctx, host); err != nil {
228 t.Fatal(err)
229 }
230 obs, _ := hostclaim.Read(f.ctx, f.store, host)
231 if !obs.Claim.Free() || len(obs.Claim.CancelledCreates) != 1 || len(obs.Claim.RetiredIncarnations) != 1 {
232 t.Fatalf("cancel: %+v", obs.Claim)
233 }
234 f.now = f.now.Add(2 * time.Hour)
235 if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
236 t.Fatal(err)
237 }
238 late, err := ReadRecord(f.ctx, f.store, r.Holder.Key)
239 if err != nil {
240 t.Fatal(err)
241 }
242 removed, err := RemoveOrphan(f.ctx, f.store, late)
243 if err != nil || !removed {
244 t.Fatalf("late orphan: %t %v", removed, err)
245 }
246}
247func TestCreateRecoveryReadErrorsNeverCancel(t *testing.T) {
248 f := newFixture(t)
249 r := f.record(t, "shop.bab.example.com")
250 before, _ := hostclaim.Read(f.ctx, f.store, r.Holder.Host)
251 obs, err := hostclaim.Update(f.ctx, f.store, before, hostclaim.BeginCreate(r.Holder, r.Body, "create", "test", f.now))
252 if err != nil {
253 t.Fatal(err)
254 }
255 f.now = f.now.Add(6 * time.Minute)
256 f.store.failGet = r.Holder.Key
257 if err := f.c.RecoverCreate(f.ctx, r.Holder.Host); err == nil {
258 t.Fatal("read error accepted")
259 }
260 got, _ := hostclaim.Read(f.ctx, f.store, r.Holder.Host)
261 if got.ETag != obs.ETag {
262 t.Fatal("acted after read error")
263 }
264 f.store.failGet = ""
265 if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
266 t.Fatal(err)
267 }
268 if err := f.c.RecoverCreate(f.ctx, r.Holder.Host); err != nil {
269 t.Fatal(err)
270 }
271 got, _ = hostclaim.Read(f.ctx, f.store, r.Holder.Host)
272 if got.Claim.Op.Kind != hostclaim.KindServing || got.Claim.Holder.Stamp != got.Claim.Gen {
273 t.Fatal("not completed")
274 }
275}
276func TestRetireWaitsForExactRecordAndAuthority(t *testing.T) {
277 f := newFixture(t)
278 obs := f.serving(t, "shop.bab.example.com")
279 key := obs.Claim.Holder.Key
280 deleting, err := f.c.BeginRetire(f.ctx, key, "test", obs.Claim.Holder.Incarnation)
281 if err != nil {
282 t.Fatal(err)
283 }
284 f.auth.err = ErrPending
285 if err := f.c.Retire(f.ctx, obs.Host, deleting.Claim.Op.ID); !errors.Is(err, ErrPending) {
286 t.Fatal(err)
287 }
288 if _, _, err := f.store.Get(f.ctx, key); err != nil {
289 t.Fatal("record deleted before barrier")
290 }
291 f.auth.err = nil
292 f.store.beforeDelete = func(k, e string) error { return release.ErrCASConflict }
293 if err := f.c.Retire(f.ctx, obs.Host, deleting.Claim.Op.ID); !errors.Is(err, release.ErrCASConflict) {
294 t.Fatal(err)
295 }
296 current, _ := hostclaim.Read(f.ctx, f.store, obs.Host)
297 if current.Claim.Op.Kind != hostclaim.KindDeleting {
298 t.Fatal("parked after 412")
299 }
300 f.store.beforeDelete = nil
301 if err := f.c.Retire(f.ctx, obs.Host, deleting.Claim.Op.ID); !errors.Is(err, ErrPending) {
302 t.Fatal(err)
303 }
304 current, _ = hostclaim.Read(f.ctx, f.store, obs.Host)
305 if current.Claim.Op.Kind != hostclaim.KindParking || len(current.Claim.Obligations) != 2 {
306 t.Fatalf("parking: %+v", current.Claim)
307 }
308 if err := f.c.Free(f.ctx, obs.Host, deleting.Claim.Op.ID); err == nil {
309 t.Fatal("freed with chores")
310 }
311 for _, ob := range current.Claim.Obligations {
312 var err error
313 current, err = hostclaim.Update(f.ctx, f.store, current, hostclaim.CompleteObligation(ob.Op, ob.ID))
314 if err != nil {
315 t.Fatal(err)
316 }
317 }
318 if err := f.c.Free(f.ctx, obs.Host, deleting.Claim.Op.ID); err != nil {
319 t.Fatal(err)
320 }
321 if f.store.bareDeletes != 0 {
322 t.Fatal("bare delete")
323 }
324}
325func TestAdoptionIsNotOrphanAndHasDiscovery(t *testing.T) {
326 f := newFixture(t)
327 r := f.record(t, "shop.bab.example.com")
328 if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
329 t.Fatal(err)
330 }
331 read, err := ReadRecord(f.ctx, f.store, r.Holder.Key)
332 if err != nil {
333 t.Fatal(err)
334 }
335 if removed, err := RemoveOrphan(f.ctx, f.store, read); err != nil || removed {
336 t.Fatal("legacy treated as orphan")
337 }
338 obs, err := f.c.Adopt(f.ctx, read)
339 if err != nil {
340 t.Fatal(err)
341 }
342 if obs.Claim.Touched.Complete || len(obs.Claim.Obligations) != 2 || obs.Claim.Holder.Location != "instances/"+r.Holder.Host || obs.Claim.Holder.WriterBox != "box-a" {
343 t.Fatalf("adopt: %+v", obs.Claim)
344 }
345 if _, err := f.c.BeginRetire(f.ctx, r.Holder.Key, "test", obs.Claim.Holder.Incarnation); err == nil {
346 t.Fatal("delete overlapped discovery")
347 }
348}
349func TestPurgeJournalPhasesAndFenceRetention(t *testing.T) {
350 f := newFixture(t)
351 p, _, err := ReservePurge(f.ctx, f.store, "bab", "jam", "purge")
352 if err != nil || p.Phase != "reserved" {
353 t.Fatal(err)
354 }
355 if _, _, err := ReservePurge(f.ctx, f.store, "bab", "jam", "other"); !errors.Is(err, ErrPending) {
356 t.Fatal(err)
357 }
358 if err := RemovePurge(f.ctx, f.store, "bab", "jam", "purge"); !errors.Is(err, ErrPending) {
359 t.Fatal(err)
360 }
361 if err := AdvancePurge(f.ctx, f.store, "bab", "jam", "purge", "reserved", "done"); err == nil {
362 t.Fatal("skipped phases")
363 }
364 for i := 0; i < len(phases)-1; i++ {
365 if err := AdvancePurge(f.ctx, f.store, "bab", "jam", "purge", phases[i], phases[i+1]); err != nil {
366 t.Fatal(err)
367 }
368 }
369 if err := RemovePurge(f.ctx, f.store, "bab", "jam", "purge"); err != nil {
370 t.Fatal(err)
371 }
372}
373func TestPurgeRetiresUnderOwnOperation(t *testing.T) {
374 f := newFixture(t)
375 obs := f.serving(t, "shop.bab.example.com")
376 rec, err := ReadRecord(f.ctx, f.store, obs.Claim.Holder.Key)
377 if err != nil {
378 t.Fatal(err)
379 }
380 obs, err = hostclaim.Update(f.ctx, f.store, obs, BeginPurge("purge", "test", f.now, []string{"ie"}, []string{"box-a"}, ""))
381 if err != nil {
382 t.Fatal(err)
383 }
384 obs, err = hostclaim.Update(f.ctx, f.store, obs, PurgeRecordIntent("purge", rec.Holder.Key, rec.ETag))
385 if err != nil {
386 t.Fatal(err)
387 }
388 f.auth.err = ErrPending
389 if err := f.c.Retire(f.ctx, obs.Host, "purge"); !errors.Is(err, ErrPending) {
390 t.Fatal(err)
391 }
392 f.auth.err = nil
393 if err := f.c.Retire(f.ctx, obs.Host, "purge"); !errors.Is(err, ErrPending) {
394 t.Fatal(err)
395 }
396 got, _ := hostclaim.Read(f.ctx, f.store, obs.Host)
397 if got.Claim.Op.Kind != hostclaim.KindParking || got.Claim.Op.ID != "purge" {
398 t.Fatal("purge identity lost")
399 }
400}
401func TestPrepareRejectsCallerTimeAndSidecarIdentity(t *testing.T) {
402 f := newFixture(t)
403 if _, err := PrepareInstance(provision.Record{CreatedAt: f.now}, f.now, "ie"); err == nil {
404 t.Fatal("accepted caller time")
405 }
406 r, err := PrepareSidecar(provision.SidecarRecord{Account: "bab", App: "jam", Name: "job", Host: provision.SidecarHost("job", "jam", "bab"), Region: "ie"}, f.now, "ie")
407 if err != nil {
408 t.Fatal(err)
409 }
410 before, _ := hostclaim.Read(f.ctx, f.store, r.Holder.Host)
411 obs, err := f.c.Attempt(f.ctx, before, r, "sidecar", "test")
412 if err != nil {
413 t.Fatal(err)
414 }
415 if obs.Claim.Holder.Family != "sidecar" {
416 t.Fatal("sidecar became instance")
417 }
418}
419func TestRecoveryFreshReadsAndRequiredHooks(t *testing.T) {
420 f := newFixture(t)
421 if err := (Recovery{Controller: f.c}).Pass(f.ctx); !errors.Is(err, ErrContract) {
422 t.Fatal(err)
423 }
424 for _, tc := range []struct {
425 name string
426 replace bool
427 wantCalls int
428 }{
429 {name: "advanced purge", replace: true, wantCalls: 1},
430 {name: "released purge", wantCalls: 0},
431 } {
432 t.Run(tc.name, func(t *testing.T) {
433 f := newFixture(t)
434 host := "shop.bab.example.com"
435 before, err := hostclaim.Read(f.ctx, f.store, host)
436 if err != nil {
437 t.Fatal(err)
438 }
439 before, err = hostclaim.Update(f.ctx, f.store, before, FencePurge("purge-old", f.now))
440 if err != nil {
441 t.Fatal(err)
442 }
443 startedAt := f.now
444 f.now = f.now.Add(6 * time.Minute)
445 key, err := hostclaim.Key(host)
446 if err != nil {
447 t.Fatal(err)
448 }
449 var fresh hostclaim.Observed
450 listCalls := 0
451 f.store.afterList = func(prefix string, keys []string) error {
452 listCalls++
453 if prefix != "control/hosts/" || len(keys) != 1 || keys[0] != key {
454 t.Fatalf("discovery = %q %v, want the populated claim", prefix, keys)
455 }
456 var err error
457 fresh, err = hostclaim.Update(f.ctx, f.store.Store, before, ReleasePurgeFence("purge-old"))
458 if err == nil && tc.replace {
459 fresh, err = hostclaim.Update(f.ctx, f.store.Store, fresh, FencePurge("purge-new", startedAt))
460 }
461 return err
462 }
463 purgeCalls, recordSweeps, purgeSweeps, admissionSweeps := 0, 0, 0, 0
464 r := Recovery{
465 Controller: f.c,
466 Purge: func(_ context.Context, got hostclaim.Observed) error {
467 purgeCalls++
468 if !tc.replace {
469 t.Fatal("dispatched a purge released after discovery")
470 }
471 if !reflect.DeepEqual(got, fresh) {
472 t.Fatalf("dispatched stale claim: gen=%d op=%q etag=%q; want gen=%d op=%q etag=%q", got.Claim.Gen, got.Claim.Op.ID, got.ETag, fresh.Claim.Gen, fresh.Claim.Op.ID, fresh.ETag)
473 }
474 return nil
475 },
476 Placement: func(context.Context, hostclaim.Observed) error {
477 t.Fatal("dispatched placement for a purge claim")
478 return nil
479 },
480 SweepRecords: func(context.Context) error { recordSweeps++; return nil },
481 SweepPurges: func(context.Context) error { purgeSweeps++; return nil },
482 SweepAdmissions: func(context.Context) error { admissionSweeps++; return nil },
483 }
484 if err := r.Pass(f.ctx); err != nil {
485 t.Fatal(err)
486 }
487 if listCalls != 1 || fresh.Claim.Gen <= before.Claim.Gen || fresh.ETag == before.ETag {
488 t.Fatal("claim did not advance between discovery and recovery")
489 }
490 if purgeCalls != tc.wantCalls {
491 t.Fatalf("purge calls = %d, want %d", purgeCalls, tc.wantCalls)
492 }
493 if recordSweeps != 1 || purgeSweeps != 1 || admissionSweeps != 1 {
494 t.Fatalf("sweeps: records=%d purges=%d admissions=%d; want one each", recordSweeps, purgeSweeps, admissionSweeps)
495 }
496 })
497 }
498}
499
500func TestAppsPreserveLegacyPrefixesAndFindClaimlessPurge(t *testing.T) {
501 f := newFixture(t)
502 for key, body := range map[string][]byte{
503 release.LegacyChannelKey("my-app", "stable"): []byte("legacy channel"),
504 release.LegacyChannelKey("myapp", "stable"): []byte("legacy channel"),
505 release.LegacyManifestKey("myapp", "v1"): []byte("legacy manifest"),
506 release.LegacyArtifactKey("myapp", "v1", "app"): []byte("legacy artifact"),
507 } {
508 if _, err := f.store.Put(f.ctx, key, body, ""); err != nil {
509 t.Fatal(err)
510 }
511 }
512 if _, _, err := ReservePurge(f.ctx, f.store, "bab", "doomed", "purge"); err != nil {
513 t.Fatal(err)
514 }
515 hosts, err := ListClaims(f.ctx, f.store)
516 if err != nil || len(hosts) != 0 {
517 t.Fatalf("claims = %v, %v; want none", hosts, err)
518 }
519 if err := f.c.SweepAllRecords(f.ctx); err != nil {
520 t.Fatalf("record sweep with legacy prefixes: %v", err)
521 }
522 var replayed []App
523 err = SweepAllPurges(f.ctx, f.store, func(_ context.Context, app App, p Purge) error {
524 if p.ID != "purge" {
525 t.Fatalf("purge ID = %q", p.ID)
526 }
527 replayed = append(replayed, app)
528 return nil
529 })
530 if err != nil {
531 t.Fatal(err)
532 }
533 if len(replayed) != 1 || replayed[0] != (App{Account: "bab", Name: "doomed"}) {
534 t.Fatalf("replayed = %+v", replayed)
535 }
536}
537
538func TestOrphanConditionalDeleteCannotEraseReplacement(t *testing.T) {
539 f := newFixture(t)
540 r := f.record(t, "shop.bab.example.com")
541 if _, err := f.store.Put(f.ctx, r.Holder.Key, r.Body, ""); err != nil {
542 t.Fatal(err)
543 }
544 old, err := ReadRecord(f.ctx, f.store, r.Holder.Key)
545 if err != nil {
546 t.Fatal(err)
547 }
548 absent, _ := hostclaim.Read(f.ctx, f.store, r.Holder.Host)
549 if _, err := hostclaim.Update(f.ctx, f.store, absent, func(*hostclaim.Claim) error { return nil }); err != nil {
550 t.Fatal(err)
551 }
552 replacement := append([]byte(nil), r.Body...)
553 replacement = append(replacement, ' ')
554 f.store.beforeDelete = func(key, etag string) error {
555 _, err := f.store.Store.Put(f.ctx, key, replacement, etag)
556 return err
557 }
558 if removed, err := RemoveOrphan(f.ctx, f.store, old); removed || !errors.Is(err, release.ErrCASConflict) {
559 t.Fatalf("removed=%t err=%v", removed, err)
560 }
561 if _, _, err := f.store.Get(f.ctx, r.Holder.Key); err != nil {
562 t.Fatal("replacement lost")
563 }
564}
565
566func TestPlacementStartKeepsInstallationAndExcludesDelete(t *testing.T) {
567 f := newFixture(t)
568 obs := f.serving(t, "shop.bab.example.com")
569 stamp := obs.Claim.Holder.Stamp
570 placed, err := hostclaim.Update(f.ctx, f.store, obs, BeginPlacement("place", "test", "fleet/customer", f.now, []string{"ie", "us"}, []string{"box-a", "box-b"}))
571 if err != nil {
572 t.Fatal(err)
573 }
574 if placed.Claim.Op.Exec != obs.Claim.Op.Exec || placed.Claim.Holder.Stamp != stamp || placed.Claim.Op.Intent["target"] != "fleet/customer" {
575 t.Fatal("placement changed authority")
576 }
577 rec, err := ReadRecord(f.ctx, f.store, obs.Claim.Holder.Key)
578 if err != nil {
579 t.Fatal(err)
580 }
581 if _, err := hostclaim.Update(f.ctx, f.store, placed, hostclaim.BeginDelete("delete", "test", rec.ETag, f.now, nil, nil, "")); err == nil {
582 t.Fatal("delete overlapped placement")
583 }
584}
585
586func TestFenceReleasePreservesHistoryAndRefusesAnotherPurge(t *testing.T) {
587 f := newFixture(t)
588 before, _ := hostclaim.Read(f.ctx, f.store, "shop.bab.example.com")
589 obs, err := hostclaim.Update(f.ctx, f.store, before, func(c *hostclaim.Claim) error {
590 c.Lost = []string{"lost-box"}
591 c.QuietAfter = map[string]time.Time{"ie": f.now.Add(time.Hour)}
592 return FencePurge("purge", f.now)(c)
593 })
594 if err != nil {
595 t.Fatal(err)
596 }
597 if _, err := hostclaim.Update(f.ctx, f.store, obs, ReleasePurgeFence("other")); err == nil {
598 t.Fatal("released other's fence")
599 }
600 obs, err = hostclaim.Update(f.ctx, f.store, obs, ReleasePurgeFence("purge"))
601 if err != nil {
602 t.Fatal(err)
603 }
604 if !obs.Claim.Free() || len(obs.Claim.Lost) != 1 || obs.Claim.QuietAfter["ie"].IsZero() {
605 t.Fatal("lost history")
606 }
607}
608
609type repairAuthority struct {
610 Authority
611 afterAdoption func()
612}
613
614func (a repairAuthority) Adoption(ctx context.Context, r Record) (string, string, []string, []string, error) {
615 region, box, regions, boxes, err := a.Authority.Adoption(ctx, r)
616 if a.afterAdoption != nil {
617 a.afterAdoption()
618 }
619 return region, box, regions, boxes, err
620}
621
622func TestAdoptLegacyRepairRequiresCensusIdentityAndFreshRecord(t *testing.T) {
623 for _, scenario := range []string{"wrong-executor", "not-serving", "owner-not-touched", "changed-record", "missing-record", "record-read-error"} {
624 t.Run(scenario, func(t *testing.T) {
625 f := newFixture(t)
626 before := f.serving(t, "legacy.example.com")
627 var err error
628 before, err = hostclaim.Update(f.ctx, f.store, before, func(c *hostclaim.Claim) error {
629 switch scenario {
630 case "wrong-executor":
631 c.Op.Exec = "box-b"
632 c.Touched.Boxes = append(c.Touched.Boxes, "box-b")
633 case "not-serving":
634 c.Op.Kind, c.Op.ID = hostclaim.KindMoving, "move"
635 case "owner-not-touched":
636 c.Touched.Boxes = []string{"box-b"}
637 }
638 return nil
639 })
640 if err != nil {
641 t.Fatal(err)
642 }
643 r, err := ReadRecord(f.ctx, f.store, before.Claim.Holder.Key)
644 if err != nil {
645 t.Fatal(err)
646 }
647 var body provision.Record
648 if err := json.Unmarshal(r.Body, &body); err != nil {
649 t.Fatal(err)
650 }
651 body.Acquisition = "ownership-acquisition"
652 raw, err := json.Marshal(body)
653 if err != nil {
654 t.Fatal(err)
655 }
656 if _, err := f.store.Put(f.ctx, r.Holder.Key, raw, r.ETag); err != nil {
657 t.Fatal(err)
658 }
659 r, err = ReadRecord(f.ctx, f.store, r.Holder.Key)
660 if err != nil {
661 t.Fatal(err)
662 }
663 want := ErrIdentity
664 if scenario == "changed-record" {
665 want = ErrPending
666 } else if scenario == "missing-record" {
667 want = os.ErrNotExist
668 }
669 f.c.Authority = repairAuthority{Authority: f.auth, afterAdoption: func() {
670 switch scenario {
671 case "changed-record":
672 if _, err := f.store.Put(f.ctx, r.Holder.Key, append(raw, ' '), r.ETag); err != nil {
673 t.Fatal(err)
674 }
675 case "missing-record":
676 if err := f.store.DeleteIfMatch(f.ctx, r.Holder.Key, r.ETag); err != nil {
677 t.Fatal(err)
678 }
679 case "record-read-error":
680 f.store.failGet = r.Holder.Key
681 }
682 }}
683 writes := 0
684 f.store.beforePut = func(string) error { writes++; return nil }
685 _, err = f.c.Adopt(f.ctx, r)
686 if scenario == "record-read-error" {
687 if err == nil || err.Error() != "read failed" {
688 t.Errorf("repair error = %v, want record read failure", err)
689 }
690 } else if !errors.Is(err, want) {
691 t.Errorf("repair error = %v, want %v", err, want)
692 }
693 if scenario == "changed-record" {
694 writes--
695 }
696 after, readErr := hostclaim.Read(f.ctx, f.store, before.Host)
697 if readErr != nil || writes != 0 || after.ETag != before.ETag || !reflect.DeepEqual(after.Claim, before.Claim) {
698 t.Fatalf("refused repair changed claim: writes=%d, read=%v", writes, readErr)
699 }
700 })
701 }
702}
703
704func TestCompleteMetadataPlacementPreservesExecution(t *testing.T) {
705 claim := hostclaim.Claim{Holder: &hostclaim.Holder{Incarnation: "old", Stamp: 42}, Gen: 50,
706 Op: hostclaim.Op{Kind: hostclaim.KindPlacing, ID: "place", Exec: "box", Participants: []string{"box"}, Intent: map[string]string{"app_id": "app"}},
707 Obligations: []hostclaim.Obligation{{ID: "keep", Op: "old", Kind: "park-local", Box: "other", Owner: "box:other"}}}
708 if err := CompleteMetadataPlacement("wrong", "app")(&claim); err == nil {
709 t.Fatal("wrong operation accepted")
710 }
711 if err := CompleteMetadataPlacement("place", "wrong")(&claim); err == nil {
712 t.Fatal("wrong app accepted")
713 }
714 if err := CompleteMetadataPlacement("place", "app")(&claim); err != nil {
715 t.Fatal(err)
716 }
717 if claim.Op.Kind != hostclaim.KindServing || claim.Op.Exec != "box" || claim.Holder.Stamp != 42 || len(claim.Obligations) != 1 || claim.Gen != 50 {
718 t.Fatalf("authority changed: %+v", claim)
719 }
720}
721