carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostops
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "reflect"
8 "strings"
9 "testing"
10 "time"
11
12 "github.com/carlosframework/platform/internal/hostclaim"
13 "github.com/carlosframework/platform/internal/provision"
14 "github.com/carlosframework/platform/internal/release"
15)
16
17// The J17 console move grant: a console-initiated cut, abort or retry is one
18// typed ControlWriter transition bound to the exact claim, registration and
19// record the door read. These tests drive it adversarially.
20
21// newJ17World is newC7World's moving claim before the seal, with the record
22// at ready (pinned inputs), the move registration in place and the claim's
23// obligations empty.
24func newJ17World(t *testing.T, mutate func(*hostclaim.Claim, *provision.Record, *AppAdmission)) *c7World {
25 t.Helper()
26 return newC7World(t, func(c *hostclaim.Claim, rec *provision.Record, adm *AppAdmission) {
27 c.Op.Phase = hostclaim.PhaseMoving
28 c.Obligations = nil
29 rec.Move.State, rec.Move.CutPhase = provision.MoveReady, ""
30 rec.Move.Inputs = &provision.MoveInputs{Release: "r1", BundleETags: map[string]string{"b": "e"}}
31 if mutate != nil {
32 mutate(c, rec, adm)
33 }
34 })
35}
36
37func j17Observe(t *testing.T, w *c7World) ConsoleMoveObservation {
38 t.Helper()
39 o, err := ObserveConsoleMove(context.Background(), w.raw, "bab", "app", w.host)
40 if err != nil {
41 t.Fatal(err)
42 }
43 return o
44}
45
46func j17Record(t *testing.T, w *c7World) (provision.Record, string) {
47 t.Helper()
48 rec, tag, err := provision.GetRecordETag(context.Background(), w.raw, "bab", "app", w.host)
49 if err != nil {
50 t.Fatal(err)
51 }
52 return *rec, tag
53}
54
55func TestJ17ObservationBindsExactIdentities(t *testing.T) {
56 w := newJ17World(t, nil)
57 o := j17Observe(t, w)
58 _, tag := j17Record(t, w)
59 claim := w.claim(t)
60 if o.Account != "bab" || o.App != "app" || o.AppID != "canonical" || o.Host != w.host || o.Incarnation != w.inc || o.OpID != "move" ||
61 o.ClaimETag != claim.ETag || o.RecordETag != tag || o.RecordSeq != 0 {
62 t.Fatalf("observation = %+v", o)
63 }
64 // Another app's door cannot observe this host's move.
65 if _, err := ObserveConsoleMove(context.Background(), w.raw, "bab", "other", w.host); !errors.Is(err, ErrIdentity) {
66 t.Fatalf("cross-app observation = %v", err)
67 }
68}
69
70func TestJ17CutAbortRetryWriteOnlyTheirTypedTransition(t *testing.T) {
71 ctx := context.Background()
72 for _, tc := range []struct {
73 name string
74 action ConsoleMoveAction
75 setup func(*hostclaim.Claim, *provision.Record, *AppAdmission)
76 want func(*provision.Record)
77 }{
78 {"cut", ConsoleMoveCut, nil, func(r *provision.Record) {
79 if err := provision.MoveGuardedCut(provision.MoveGuard{ID: "move", Incarnation: r.Move.Incarnation})(r); err != nil {
80 panic(err)
81 }
82 }},
83 {"abort", ConsoleMoveAbort, nil, func(r *provision.Record) {
84 if err := provision.MoveGuardedAbort(provision.MoveGuard{ID: "move", Incarnation: r.Move.Incarnation})(r); err != nil {
85 panic(err)
86 }
87 }},
88 {"retry", ConsoleMoveRetry, func(c *hostclaim.Claim, rec *provision.Record, _ *AppAdmission) {
89 c.Op.Kind, c.Op.Phase, c.Op.Exec = hostclaim.KindRestoring, hostclaim.PhaseProbing, "source"
90 c.Op.Intent["rollback"] = "rb_abcdefghijklmnopqrstuvwxyz"
91 rec.Region = "ie"
92 rec.Move.State, rec.Move.StuckReason = provision.MoveStuck, provision.StuckRecoveryProbeAttempts
93 rec.Move.Rollback = &provision.MoveRollback{ID: "rb_abcdefghijklmnopqrstuvwxyz", TargetPhase: provision.RbTargetReleased, SourcePhase: provision.RbSourceProbing, StuckAt: provision.RbSourceProbing}
94 }, func(r *provision.Record) {
95 if err := provision.MoveRetryRecovery(provision.MoveGuard{ID: "move", Incarnation: r.Move.Incarnation})(r); err != nil {
96 panic(err)
97 }
98 }},
99 } {
100 t.Run(tc.name, func(t *testing.T) {
101 w := newJ17World(t, tc.setup)
102 before, _ := j17Record(t, w)
103 claim := w.claim(t)
104 got, err := w.control.ConsoleMove(ctx, j17Observe(t, w), tc.action)
105 if err != nil {
106 t.Fatal(err)
107 }
108 after, _ := j17Record(t, w)
109 want := before
110 mv := *before.Move
111 want.Move = &mv
112 tc.want(&want)
113 want.Move.UpdatedAt, after.Move.UpdatedAt = time.Time{}, time.Time{}
114 want.RecordSeq = before.RecordSeq + 1
115 if !reflect.DeepEqual(want, after) {
116 t.Fatalf("record:\n%+v\nwant\n%+v", after.Move, want.Move)
117 }
118 if got == nil || got.Move == nil || got.Move.State != after.Move.State || got.RecordSeq != after.RecordSeq {
119 t.Fatalf("returned record = %+v", got)
120 }
121 if c := w.claim(t); c.ETag != claim.ETag {
122 t.Fatal("a console record transition wrote the claim")
123 }
124 })
125 }
126}
127
128// Expiry and closed admission do not revoke an accepted move: its console
129// transitions still drain it.
130func TestJ17ExpiredRegistrationAndClosedAdmissionStillDrain(t *testing.T) {
131 w := newJ17World(t, func(_ *hostclaim.Claim, _ *provision.Record, adm *AppAdmission) {
132 r := adm.Registrations["move"]
133 r.Expires = time.Now().Add(-time.Hour)
134 adm.Registrations["move"] = r
135 adm.State = "closed"
136 })
137 w.clock = time.Now().Add(time.Hour)
138 if _, err := w.control.ConsoleMove(context.Background(), j17Observe(t, w), ConsoleMoveAbort); err != nil {
139 t.Fatal(err)
140 }
141}
142
143func TestJ17RefusesInvalidRegistrationBeforeAnyWrite(t *testing.T) {
144 for name, mutate := range map[string]func(*AppAdmission){
145 "missing": func(a *AppAdmission) { delete(a.Registrations, "move") },
146 "other kind": func(a *AppAdmission) { r := a.Registrations["move"]; r.Kind = "start"; a.Registrations["move"] = r },
147 "other host": func(a *AppAdmission) {
148 r := a.Registrations["move"]
149 r.Host = "other.example"
150 a.Registrations["move"] = r
151 },
152 "other inc": func(a *AppAdmission) {
153 r := a.Registrations["move"]
154 r.Incarnation = strings.Repeat("f", 64)
155 a.Registrations["move"] = r
156 },
157 "other epoch": func(a *AppAdmission) { a.Epoch = 2 },
158 "other appid": func(a *AppAdmission) { r := a.Registrations["move"]; r.AppID = "other"; a.Registrations["move"] = r },
159 } {
160 t.Run(name, func(t *testing.T) {
161 w := newJ17World(t, func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) { mutate(a) })
162 before, tag := j17Record(t, w)
163 if _, err := w.control.ConsoleMove(context.Background(), j17Observe(t, w), ConsoleMoveAbort); !errors.Is(err, ErrPending) {
164 t.Fatalf("err = %v", err)
165 }
166 after, tag2 := j17Record(t, w)
167 if tag != tag2 || !reflect.DeepEqual(before, after) {
168 t.Fatal("refused transition wrote the record")
169 }
170 })
171 }
172}
173
174// Every field of the observation is authority-bearing: a changed one is
175// refused before any write, whatever else still matches.
176func TestJ17RefusesTamperedObservation(t *testing.T) {
177 for name, tamper := range map[string]func(*ConsoleMoveObservation){
178 "account": func(o *ConsoleMoveObservation) { o.Account = "zzz" },
179 "app": func(o *ConsoleMoveObservation) { o.App = "other" },
180 "app id": func(o *ConsoleMoveObservation) { o.AppID = "other" },
181 "host": func(o *ConsoleMoveObservation) { o.Host = "other.example" },
182 "incarnation": func(o *ConsoleMoveObservation) { o.Incarnation = strings.Repeat("f", 64) },
183 "op": func(o *ConsoleMoveObservation) { o.OpID = "move-2" },
184 "claim etag": func(o *ConsoleMoveObservation) { o.ClaimETag = "\"stale\"" },
185 "record etag": func(o *ConsoleMoveObservation) { o.RecordETag = "\"stale\"" },
186 "record seq": func(o *ConsoleMoveObservation) { o.RecordSeq = 9 },
187 "empty": func(o *ConsoleMoveObservation) { *o = ConsoleMoveObservation{} },
188 } {
189 t.Run(name, func(t *testing.T) {
190 w := newJ17World(t, nil)
191 o := j17Observe(t, w)
192 tamper(&o)
193 _, tag := j17Record(t, w)
194 if _, err := w.control.ConsoleMove(context.Background(), o, ConsoleMoveAbort); err == nil {
195 t.Fatal("tampered observation accepted")
196 }
197 if _, tag2 := j17Record(t, w); tag != tag2 {
198 t.Fatal("refused transition wrote the record")
199 }
200 })
201 }
202}
203
204func TestJ17RefusesUnknownOrChangedAction(t *testing.T) {
205 w := newJ17World(t, nil)
206 o := j17Observe(t, w)
207 _, tag := j17Record(t, w)
208 for _, a := range []ConsoleMoveAction{"", "declare-lost", "rollback", "cut ", "CUT", "abort;cut"} {
209 if _, err := w.control.ConsoleMove(context.Background(), o, a); !errors.Is(err, ErrIdentity) {
210 t.Fatalf("action %q: %v", a, err)
211 }
212 }
213 // A legal action the record's state does not admit: the provision
214 // refusal is returned unchanged and nothing is written.
215 if _, err := w.control.ConsoleMove(context.Background(), o, ConsoleMoveRetry); err == nil {
216 t.Fatal("retry of a moving claim accepted")
217 }
218 if _, tag2 := j17Record(t, w); tag != tag2 {
219 t.Fatal("refused action wrote the record")
220 }
221}
222
223// The claim's operation decides which actions exist at all.
224func TestJ17ActionMustMatchTheClaimOperation(t *testing.T) {
225 w := newJ17World(t, func(c *hostclaim.Claim, _ *provision.Record, _ *AppAdmission) {
226 c.Op.Phase = hostclaim.PhaseSealed
227 })
228 if _, err := w.control.ConsoleMove(context.Background(), j17Observe(t, w), ConsoleMoveAbort); !errors.Is(err, hostclaim.ErrWrongState) {
229 t.Fatalf("abort of a sealed cut = %v", err)
230 }
231 w = newJ17World(t, func(c *hostclaim.Claim, rec *provision.Record, _ *AppAdmission) {
232 c.Op.Kind = hostclaim.KindLanding
233 rec.Move.State = provision.MoveStuck
234 })
235 if _, err := w.control.ConsoleMove(context.Background(), j17Observe(t, w), ConsoleMoveRetry); !errors.Is(err, hostclaim.ErrWrongState) {
236 t.Fatalf("retry of a landing = %v", err)
237 }
238}
239
240func TestJ17CrossHostAndCrossOp(t *testing.T) {
241 ctx := context.Background()
242 a := newJ17World(t, nil)
243 b := newJ17World(t, nil)
244 // b's observation names b's host, not a's: a's writer cannot act on it
245 // because a's bucket has no such claim; and a forged pairing of a's host
246 // with b's identities is refused.
247 ob := j17Observe(t, b)
248 oa := j17Observe(t, a)
249 ob.Host = oa.Host
250 if _, err := a.control.ConsoleMove(ctx, ob, ConsoleMoveAbort); err == nil {
251 t.Fatal("cross-host observation accepted")
252 }
253 // The claim names another op (a successor move under the same
254 // registration name is impossible; another op id is refused).
255 c := newJ17World(t, nil)
256 oc := j17Observe(t, c)
257 if _, err := hostclaim.Update(ctx, c.raw, c.claim(t), func(cl *hostclaim.Claim) error {
258 cl.Op.Intent["admission_id"] = "other"
259 return nil
260 }); err != nil {
261 t.Fatal(err)
262 }
263 if _, err := ObserveConsoleMove(ctx, c.raw, "bab", "app", c.host); !errors.Is(err, ErrIdentity) {
264 t.Fatalf("observed an op whose admission id is not its own id: %v", err)
265 }
266 oc.ClaimETag = c.claim(t).ETag
267 if _, err := c.control.ConsoleMove(ctx, oc, ConsoleMoveAbort); !errors.Is(err, ErrIdentity) {
268 t.Fatalf("an op whose admission id is not its own id = %v", err)
269 }
270 d := newJ17World(t, func(_ *hostclaim.Claim, rec *provision.Record, _ *AppAdmission) {
271 rec.Move.ID = "move-2"
272 })
273 if _, err := ObserveConsoleMove(ctx, d.raw, "bab", "app", d.host); !errors.Is(err, ErrIdentity) {
274 t.Fatalf("record for another move observed: %v", err)
275 }
276}
277
278// A successor incarnation holding the host refuses the predecessor's
279// observation even when the move id is reused.
280func TestJ17SuccessorIncarnationRefused(t *testing.T) {
281 ctx := context.Background()
282 w := newJ17World(t, nil)
283 o := j17Observe(t, w)
284 obs := w.claim(t)
285 if _, err := hostclaim.Update(ctx, w.raw, obs, func(c *hostclaim.Claim) error {
286 c.Holder.CreatedAt = c.Holder.CreatedAt.Add(time.Second)
287 inc, err := hostclaim.Incarnation("bab", "app", w.host, c.Holder.CreatedAt)
288 c.Holder.Incarnation = inc
289 return err
290 }); err != nil {
291 t.Fatal(err)
292 }
293 o.ClaimETag = w.claim(t).ETag
294 if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, ErrIdentity) {
295 t.Fatalf("successor = %v", err)
296 }
297}
298
299func TestJ17StaleClaimAndRecordRefused(t *testing.T) {
300 ctx := context.Background()
301 w := newJ17World(t, nil)
302 o := j17Observe(t, w)
303 // An unrelated claim write (a chore acknowledgement) since the read.
304 obs := w.claim(t)
305 if _, err := hostclaim.Update(ctx, w.raw, obs, func(*hostclaim.Claim) error { return nil }); err != nil {
306 t.Fatal(err)
307 }
308 if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, hostclaim.ErrConflict) {
309 t.Fatalf("stale claim = %v", err)
310 }
311 w = newJ17World(t, nil)
312 o = j17Observe(t, w)
313 if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), w.raw, "bab", "app", w.host, func(r *provision.Record) error { r.Move.Reason = "x"; return nil }); err != nil {
314 t.Fatal(err)
315 }
316 if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, release.ErrCASConflict) {
317 t.Fatalf("stale record = %v", err)
318 }
319}
320
321// A changed record sequence changes content-addressed ETags too. Even a
322// forged current ETag cannot make an older sequence observation valid.
323func TestJ17ForgedETagCannotHideOldSequence(t *testing.T) {
324 ctx := context.Background()
325 w := newJ17World(t, nil)
326 seqCtx := provision.WithRecordSequence(ctx)
327 if _, err := provision.SetRecordMove(seqCtx, w.raw, "bab", "app", w.host, func(r *provision.Record) error { return nil }); err != nil {
328 t.Fatal(err)
329 }
330 o := j17Observe(t, w)
331 body, tag, err := w.raw.Get(ctx, w.key)
332 if err != nil {
333 t.Fatal(err)
334 }
335 // Two writes that restore the same bytes apart from the counter.
336 for i := 0; i < 2; i++ {
337 if _, err := provision.SetRecordMove(seqCtx, w.raw, "bab", "app", w.host, func(r *provision.Record) error { return nil }); err != nil {
338 t.Fatal(err)
339 }
340 }
341 // Forge an observation whose ETag matches the current bytes but whose
342 // sequence is the earlier read.
343 _, tag3 := j17Record(t, w)
344 o.RecordETag = tag3
345 if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, release.ErrCASConflict) {
346 t.Fatalf("ABA = %v (tag %s body %d)", err, tag, len(body))
347 }
348}
349
350// A lost answer: the transition landed, the caller retries with the same
351// observation. The retry is refused and the record stays as the first
352// write left it; the door's answer is to read the state.
353func TestJ17LostAnswerRetryIsFenced(t *testing.T) {
354 ctx := context.Background()
355 w := newJ17World(t, nil)
356 o := j17Observe(t, w)
357 if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); err != nil {
358 t.Fatal(err)
359 }
360 after, tag := j17Record(t, w)
361 if _, err := w.control.ConsoleMove(ctx, o, ConsoleMoveAbort); err == nil {
362 t.Fatal("replayed observation accepted")
363 }
364 again, tag2 := j17Record(t, w)
365 if tag != tag2 || !reflect.DeepEqual(after, again) {
366 t.Fatal("replay wrote the record")
367 }
368}
369
370// The registration is checked again at the final conditional write: one
371// removed between the transition's reads and its Put refuses the Put.
372func TestJ17RegistrationCheckedAtTheFinalWrite(t *testing.T) {
373 ctx := context.Background()
374 w := newJ17World(t, nil)
375 o := j17Observe(t, w)
376 hook := &j17RecordHook{Store: w.raw, key: w.key}
377 hook.beforePut = func() {
378 a, err := ReadAdmission(ctx, w.raw, "bab", "app", "canonical")
379 if err != nil {
380 t.Fatal(err)
381 }
382 delete(a.Value.Registrations, "move")
383 b, _ := json.Marshal(a.Value)
384 if _, err := w.raw.Put(ctx, a.Key, b, a.ETag); err != nil {
385 t.Fatal(err)
386 }
387 }
388 cw, err := NewControlWriter(hook, w.pub, func() time.Time { return w.clock })
389 if err != nil {
390 t.Fatal(err)
391 }
392 _, tag := j17Record(t, w)
393 if _, err := cw.ConsoleMove(ctx, o, ConsoleMoveAbort); !errors.Is(err, ErrPending) {
394 t.Fatalf("late registration removal = %v", err)
395 }
396 if _, tag2 := j17Record(t, w); tag != tag2 {
397 t.Fatal("record written after the registration was removed")
398 }
399}
400
401// j17RecordHook runs beforePut once, before the second admission read
402// used by the final claim and registration check.
403type j17RecordHook struct {
404 release.Store
405 key string
406 reads int
407 beforePut func()
408}
409
410func (h *j17RecordHook) Get(ctx context.Context, key string) ([]byte, string, error) {
411 if strings.HasSuffix(key, "/admission.json") {
412 h.reads++
413 if h.reads == 2 && h.beforePut != nil {
414 f := h.beforePut
415 h.beforePut = nil
416 f()
417 }
418 }
419 return h.Store.Get(ctx, key)
420}
421
422// No exported method of the typed writers takes or returns a store, a
423// callback or a mutation function: there is nothing to inject.
424func TestJ17NoRawStoreOrCallbackEscapes(t *testing.T) {
425 storeType := reflect.TypeOf((*release.Store)(nil)).Elem()
426 for _, typ := range []reflect.Type{reflect.TypeOf(&ControlWriter{})} {
427 for i := 0; i < typ.NumMethod(); i++ {
428 m := typ.Method(i)
429 if !strings.Contains(m.Name, "Move") && !strings.Contains(m.Name, "Loss") {
430 continue
431 }
432 ft := m.Type
433 check := func(t2 reflect.Type) {
434 if t2.Kind() == reflect.Func || t2 == storeType || t2.Implements(storeType) {
435 t.Errorf("%s exposes %s", m.Name, t2)
436 }
437 }
438 for j := 1; j < ft.NumIn(); j++ {
439 check(ft.In(j))
440 }
441 for j := 0; j < ft.NumOut(); j++ {
442 check(ft.Out(j))
443 }
444 }
445 }
446 // The observation carries values only.
447 ot := reflect.TypeOf(ConsoleMoveObservation{})
448 for i := 0; i < ot.NumField(); i++ {
449 if k := ot.Field(i).Type.Kind(); k != reflect.String && k != reflect.Uint64 {
450 t.Errorf("observation field %s is %s", ot.Field(i).Name, k)
451 }
452 }
453}
454
455// A context carrying a forged or foreign lifecycle grant adds nothing.
456func TestJ17ContextGrantsAreIgnored(t *testing.T) {
457 ctx := context.Background()
458 w := newJ17World(t, func(_ *hostclaim.Claim, _ *provision.Record, a *AppAdmission) { delete(a.Registrations, "move") })
459 ctx = withLifecycleGrant(ctx, lifecycleGrant{kind: "recovery", w: w.control})
460 ctx = context.WithValue(ctx, lifecycleRecordKey{}, lifecycleRecordPermit{Observation: w.claim(t)})
461 if _, err := w.control.ConsoleMove(ctx, j17Observe(t, w), ConsoleMoveAbort); !errors.Is(err, ErrPending) {
462 t.Fatalf("injected grant = %v", err)
463 }
464}
465