| 1 | package hostactors |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "reflect" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/boxid" |
| 14 | "github.com/carlosframework/platform/internal/boxid/boxidtest" |
| 15 | "github.com/carlosframework/platform/internal/release" |
| 16 | "github.com/carlosframework/platform/internal/status" |
| 17 | ) |
| 18 | |
| 19 | // fakeMembership is the cloud, as the console's reader sees it: each |
| 20 | // instance's membership. err fails every call, with failIDs the request |
| 21 | // ids of the calls made; during runs inside each call (a slow describe, |
| 22 | // say). |
| 23 | type fakeMembership struct { |
| 24 | members map[string]Membership |
| 25 | err error |
| 26 | failIDs []string |
| 27 | during func() |
| 28 | calls []string |
| 29 | } |
| 30 | |
| 31 | func (f *fakeMembership) Member(_ context.Context, awsRegion, instance string) (Membership, error) { |
| 32 | f.calls = append(f.calls, awsRegion+" "+instance) |
| 33 | if f.during != nil { |
| 34 | f.during() |
| 35 | } |
| 36 | if f.err != nil { |
| 37 | return Membership{RequestIDs: f.failIDs}, f.err |
| 38 | } |
| 39 | return f.members[instance], nil |
| 40 | } |
| 41 | |
| 42 | func member(group string) Membership { |
| 43 | return Membership{GroupName: group, AvailabilityZoneRegion: "eu-west-1", LifecycleState: "InService", InstanceState: "running", Owner: "123", RequestIDs: []string{"asg-req", "ec2-req"}} |
| 44 | } |
| 45 | |
| 46 | // memberWorld is fixture()'s roster with ie's group "routers-ie", and a |
| 47 | // box (instance i-new, reference newRef) the group launched, invited, with an open |
| 48 | // attempt. |
| 49 | type memberWorld struct { |
| 50 | d Directory |
| 51 | sg release.Signer |
| 52 | o Observed |
| 53 | v boxid.Verifier |
| 54 | raw []byte |
| 55 | in EnrollmentRequest |
| 56 | cloud *fakeMembership |
| 57 | clock time.Time |
| 58 | } |
| 59 | |
| 60 | func newMemberWorld(t *testing.T, m Membership) *memberWorld { |
| 61 | t.Helper() |
| 62 | ctx := context.Background() |
| 63 | d, sg, o := fixture(t) |
| 64 | aws := boxidtest.NewSigner(t) |
| 65 | _, raw := aws.Record(t, newRef, "123", "eu-west-1", "i-new") |
| 66 | next := Clone(o.Roster) |
| 67 | next.Regions = map[string]Region{"ie": {Account: "123", AWSRegion: "eu-west-1", GroupName: "routers-ie", GroupARN: "arn:aws:autoscaling:eu-west-1:123:autoScalingGroup:x:autoScalingGroupName/routers-ie", Namespaces: []string{"platform"}, Pools: []string{"default"}}} |
| 68 | o, err := d.Replace(ctx, sg, o, next) |
| 69 | if err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | w := &memberWorld{d: d, sg: sg, v: boxid.Verifier{Account: "123", Certs: aws.Certs()}, raw: raw, clock: time.Now().UTC(), |
| 73 | cloud: &fakeMembership{members: map[string]Membership{"i-new": m}}, |
| 74 | in: EnrollmentRequest{Reference: newRef, Namespace: "platform", Label: newRef, Region: "ie", Pool: "default", AWSRegion: "eu-west-1"}} |
| 75 | if _, err := d.Invite(ctx, sg, newRef, raw); err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | o, a, err := d.BeginEnrollmentAttempt(ctx, sg, o, w.in, raw, "i-new") |
| 79 | if err != nil { |
| 80 | t.Fatal(err) |
| 81 | } |
| 82 | w.o, w.in.AttemptID, w.in.ConfirmationID = o, a.AttemptID, a.ConfirmationID |
| 83 | return w |
| 84 | } |
| 85 | |
| 86 | func (w *memberWorld) now() time.Time { return w.clock } |
| 87 | |
| 88 | func (w *memberWorld) enroll() (Observed, MembershipEvidence, error) { |
| 89 | return w.d.EnrollAutomatic(context.Background(), w.sg, w.o, w.v, w.in, w.raw, "i-new", w.cloud, w.now) |
| 90 | } |
| 91 | |
| 92 | // The console reads membership itself, once, in the attempt's AWS region, |
| 93 | // and enrolls on it; the read's request ids come back for its log. |
| 94 | func TestEnrollAutomaticReadsMembershipItself(t *testing.T) { |
| 95 | w := newMemberWorld(t, member("routers-ie")) |
| 96 | next, ev, err := w.enroll() |
| 97 | if err != nil { |
| 98 | t.Fatal(err) |
| 99 | } |
| 100 | if b := next.Roster.Boxes[newRef]; b.State != "active" || b.Group != "routers-ie" || b.Cloud.InstanceID != "i-new" { |
| 101 | t.Fatalf("roster entry %+v", b) |
| 102 | } |
| 103 | if !reflect.DeepEqual(w.cloud.calls, []string{"eu-west-1 i-new"}) || !reflect.DeepEqual(ev.RequestIDs, []string{"asg-req", "ec2-req"}) || !ev.ObservedAt.Equal(w.clock) { |
| 104 | t.Fatalf("calls %v, evidence %+v", w.cloud.calls, ev) |
| 105 | } |
| 106 | // A replay of the consumed attempt is refused before the cloud is |
| 107 | // asked again. |
| 108 | w.o = next |
| 109 | if _, _, err := w.enroll(); !errors.Is(err, ErrRoster) || len(w.cloud.calls) != 1 { |
| 110 | t.Fatalf("replay: %v, calls %v", err, w.cloud.calls) |
| 111 | } |
| 112 | if _, _, err := w.d.checkConfirmed(context.Background(), next, w.v, w.in, w.raw, "i-new", w.cloud, w.now); !errors.Is(err, ErrRoster) { |
| 113 | t.Fatalf("replayed check: %v", err) |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // Every positive answer that the instance is not a current member of the |
| 118 | // region's group refuses, and writes nothing. |
| 119 | func TestEnrollAutomaticRefusesANonMember(t *testing.T) { |
| 120 | for name, m := range map[string]Membership{ |
| 121 | "another group": member("routers-uk"), |
| 122 | "no group": {Owner: "123", InstanceState: "running", AvailabilityZoneRegion: "eu-west-1", RequestIDs: []string{"a", "b"}}, |
| 123 | "leaving the group": func() Membership { m := member("routers-ie"); m.LifecycleState = "Terminating"; return m }(), |
| 124 | "stopped": func() Membership { m := member("routers-ie"); m.InstanceState = "stopped"; return m }(), |
| 125 | "terminated": func() Membership { m := member("routers-ie"); m.InstanceState = "terminated"; return m }(), |
| 126 | "another account": func() Membership { m := member("routers-ie"); m.Owner = "999"; return m }(), |
| 127 | "another zone": func() Membership { m := member("routers-ie"); m.AvailabilityZoneRegion = "eu-west-2"; return m }(), |
| 128 | } { |
| 129 | t.Run(name, func(t *testing.T) { |
| 130 | w := newMemberWorld(t, m) |
| 131 | if _, _, err := w.d.checkConfirmed(context.Background(), w.o, w.v, w.in, w.raw, "i-new", w.cloud, w.now); !errors.Is(err, ErrNotMember) { |
| 132 | t.Fatalf("check: %v", err) |
| 133 | } |
| 134 | if _, _, err := w.enroll(); !errors.Is(err, ErrNotMember) { |
| 135 | t.Fatalf("enroll: %v", err) |
| 136 | } |
| 137 | o, err := w.d.Read(context.Background()) |
| 138 | if err != nil || o.Roster.Revision != w.o.Roster.Revision { |
| 139 | t.Fatalf("a refused enrollment wrote the roster: %v", err) |
| 140 | } |
| 141 | }) |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | // A read that fails, or is not an answer, is unknown: never a member and |
| 146 | // never a non-member. |
| 147 | func TestAMembershipReadThatIsNotAnAnswerIsUnknown(t *testing.T) { |
| 148 | for name, edit := range map[string]func(*fakeMembership){ |
| 149 | "the call fails": func(f *fakeMembership) { f.err = errors.New("Throttling") }, |
| 150 | "EC2 does not list it": func(f *fakeMembership) { |
| 151 | m := f.members["i-new"] |
| 152 | m.InstanceState, m.Owner = "", "" |
| 153 | f.members["i-new"] = m |
| 154 | }, |
| 155 | "one request id": func(f *fakeMembership) { m := f.members["i-new"]; m.RequestIDs = []string{"a"}; f.members["i-new"] = m }, |
| 156 | "a blank request id": func(f *fakeMembership) { |
| 157 | m := f.members["i-new"] |
| 158 | m.RequestIDs = []string{"a", ""} |
| 159 | f.members["i-new"] = m |
| 160 | }, |
| 161 | // A call that failed before the cloud answered has no id, so the |
| 162 | // reply is not an answer however it got this far. |
| 163 | "no id from a call": func(f *fakeMembership) { |
| 164 | m := f.members["i-new"] |
| 165 | m.RequestIDs = []string{"a", NoRequestID} |
| 166 | f.members["i-new"] = m |
| 167 | }, |
| 168 | // The ids are checked before anything the reply claims: another |
| 169 | // group with one id is still no answer, not a refusal. |
| 170 | "another group, one request id": func(f *fakeMembership) { |
| 171 | m := f.members["i-new"] |
| 172 | m.GroupName, m.RequestIDs = "routers-uk", []string{"a"} |
| 173 | f.members["i-new"] = m |
| 174 | }, |
| 175 | "no group and no ids": func(f *fakeMembership) { |
| 176 | f.members["i-new"] = Membership{} |
| 177 | }, |
| 178 | } { |
| 179 | t.Run(name, func(t *testing.T) { |
| 180 | w := newMemberWorld(t, member("routers-ie")) |
| 181 | edit(w.cloud) |
| 182 | if _, _, err := w.d.checkConfirmed(context.Background(), w.o, w.v, w.in, w.raw, "i-new", w.cloud, w.now); !errors.Is(err, ErrMembershipUnknown) || errors.Is(err, ErrNotMember) { |
| 183 | t.Fatalf("check: %v", err) |
| 184 | } |
| 185 | _, _, err := w.enroll() |
| 186 | if !errors.Is(err, ErrMembershipUnknown) || errors.Is(err, ErrNotMember) { |
| 187 | t.Fatalf("%v", err) |
| 188 | } |
| 189 | // The attempt is kept as it was, and nothing is written. |
| 190 | o, err := w.d.Read(context.Background()) |
| 191 | a := o.Roster.EnrollmentAttempts[newRef] |
| 192 | if err != nil || o.Roster.Revision != w.o.Roster.Revision || a.Consumed || a.AttemptID != w.in.AttemptID || a.ConfirmationID != w.in.ConfirmationID { |
| 193 | t.Fatalf("an unknown read moved the attempt: %v, %+v", err, a) |
| 194 | } |
| 195 | }) |
| 196 | } |
| 197 | } |
| 198 | |
| 199 | // A read that failed still logs the ids the cloud gave: the call that |
| 200 | // answered, and the failed call's own id from its error response. |
| 201 | func TestAFailedReadKeepsTheIdsTheCloudGave(t *testing.T) { |
| 202 | for name, ids := range map[string][]string{ |
| 203 | "the second call fails": {"asg-req", "ec2-err"}, |
| 204 | "the first call fails": {"asg-err"}, |
| 205 | "a call fails before the cloud answers": {"asg-req", NoRequestID}, |
| 206 | } { |
| 207 | t.Run(name, func(t *testing.T) { |
| 208 | w := newMemberWorld(t, member("routers-ie")) |
| 209 | w.cloud.err, w.cloud.failIDs = errors.New("UnauthorizedOperation"), ids |
| 210 | _, c, err := w.d.checkConfirmed(context.Background(), w.o, w.v, w.in, w.raw, "i-new", w.cloud, w.now) |
| 211 | ev := c.evidence() |
| 212 | if !errors.Is(err, ErrMembershipUnknown) || !reflect.DeepEqual(ev.RequestIDs, ids) || !ev.ObservedAt.Equal(w.clock) { |
| 213 | t.Fatalf("check: %v, evidence %+v", err, ev) |
| 214 | } |
| 215 | _, ev, err = w.enroll() |
| 216 | if !errors.Is(err, ErrMembershipUnknown) || !reflect.DeepEqual(ev.RequestIDs, ids) { |
| 217 | t.Fatalf("enroll: %v, evidence %+v", err, ev) |
| 218 | } |
| 219 | c, err = confirmMembership(context.Background(), w.cloud, w.o.Roster.Regions["ie"], "i-new", w.now) |
| 220 | if !errors.Is(err, ErrMembershipUnknown) || !reflect.DeepEqual(c.requestIDs, ids) { |
| 221 | t.Fatalf("confirmMembership: %v, %v", err, c.requestIDs) |
| 222 | } |
| 223 | }) |
| 224 | } |
| 225 | } |
| 226 | |
| 227 | // C2: the write deadline is 60 seconds from the read's anchor, taken |
| 228 | // before the describe and so never later than its completion, and a read |
| 229 | // sixty seconds old or more is refused. A describe that took exactly a |
| 230 | // minute leaves the roster as it was. |
| 231 | func TestEnrollAutomaticRefusesAReadOlderThanAMinuteAtTheWrite(t *testing.T) { |
| 232 | w := newMemberWorld(t, member("routers-ie")) |
| 233 | w.cloud.during = func() { w.clock = w.clock.Add(60 * time.Second) } |
| 234 | if _, _, err := w.enroll(); !errors.Is(err, ErrMembershipStale) { |
| 235 | t.Fatalf("slow read: %v", err) |
| 236 | } |
| 237 | o, err := w.d.Read(context.Background()) |
| 238 | if err != nil || o.Roster.Revision != w.o.Roster.Revision || o.Roster.EnrollmentAttempts[newRef].Consumed { |
| 239 | t.Fatalf("a stale read wrote the roster: %v", err) |
| 240 | } |
| 241 | // A read that was fresh when it was checked but is over a minute old |
| 242 | // by the write: here every look at the clock takes 31 seconds, so the |
| 243 | // read is 31 seconds old when checked and 62 when the roster write is |
| 244 | // about to be made. The write bound refuses it there, as unknown. |
| 245 | u := newMemberWorld(t, member("routers-ie")) |
| 246 | ticking := func() time.Time { u.clock = u.clock.Add(31 * time.Second); return u.clock } |
| 247 | if _, _, err := u.d.EnrollAutomatic(context.Background(), u.sg, u.o, u.v, u.in, u.raw, "i-new", u.cloud, ticking); !errors.Is(err, ErrWriteDeadline) || errors.Is(err, ErrMembershipStale) { |
| 248 | t.Fatalf("a read gone stale before the write: %v", err) |
| 249 | } |
| 250 | if o, err := u.d.Read(context.Background()); err != nil || o.Roster.Revision != u.o.Roster.Revision { |
| 251 | t.Fatalf("a read gone stale before the write wrote the roster: %v", err) |
| 252 | } |
| 253 | // Fifty-nine seconds is inside the minute. |
| 254 | v := newMemberWorld(t, member("routers-ie")) |
| 255 | v.cloud.during = func() { v.clock = v.clock.Add(59 * time.Second) } |
| 256 | if _, _, err := v.enroll(); err != nil { |
| 257 | t.Fatalf("a read inside the minute: %v", err) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // No exported identifier of this package carries a membership |
| 262 | // confirmation: the only one is built inside it, from a reader's answer. |
| 263 | func TestNoExportedPathTakesAConfirmation(t *testing.T) { |
| 264 | // Every exported method of Directory, EnrollAutomatic among them: the |
| 265 | // admission check itself (checkConfirmed) is not exported. |
| 266 | if _, ok := reflect.TypeOf(Directory{}).MethodByName("EnrollAutomatic"); !ok { |
| 267 | t.Fatal("no EnrollAutomatic") |
| 268 | } |
| 269 | if _, ok := reflect.TypeOf(Directory{}).MethodByName("CheckConfirmed"); ok { |
| 270 | t.Fatal("the admission check is exported again") |
| 271 | } |
| 272 | for j := 0; j < reflect.TypeOf(Directory{}).NumMethod(); j++ { |
| 273 | m := reflect.TypeOf(Directory{}).Method(j) |
| 274 | name := m.Name |
| 275 | for i := 0; i < m.Type.NumIn(); i++ { |
| 276 | in := m.Type.In(i) |
| 277 | if strings.Contains(strings.ToLower(in.Name()), "confirmation") || in == reflect.TypeOf(MembershipEvidence{}) || in == reflect.TypeOf(Membership{}) { |
| 278 | t.Fatalf("%s takes a %s", name, in) |
| 279 | } |
| 280 | } |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | // newRef is the joining box's reference: the group reference of |
| 285 | // instance i-new in account 123. |
| 286 | var newRef = GroupBoxReference("123", "i-new") |
| 287 | |
| 288 | // The ruling's first test: two group boxes never share a reference. The |
| 289 | // reference is a pseudonym of account and instance, stable, and outside |
| 290 | // the label space. |
| 291 | func TestGroupBoxReferencesDoNotCollide(t *testing.T) { |
| 292 | if GroupBoxReference("123", "i-0a61f580df1acfa65") != GroupBoxReference("123", "i-0a61f580df1acfa65") { |
| 293 | t.Fatal("not stable") |
| 294 | } |
| 295 | if GroupBoxReference("123", "i-a") == GroupBoxReference("124", "i-a") { |
| 296 | t.Fatal("the account is not in the reference") |
| 297 | } |
| 298 | seen := map[string]string{} |
| 299 | for i := 0; i < 200000; i++ { |
| 300 | inst := fmt.Sprintf("i-%017x", i) |
| 301 | ref := GroupBoxReference("123456789012", inst) |
| 302 | if len(ref) != 22 || !strings.HasPrefix(ref, "g-") || !segment(ref) { |
| 303 | t.Fatalf("%s: %q", inst, ref) |
| 304 | } |
| 305 | if other, ok := seen[ref]; ok { |
| 306 | t.Fatalf("%s and %s share %s", inst, other, ref) |
| 307 | } |
| 308 | seen[ref] = inst |
| 309 | } |
| 310 | if strings.Contains(GroupBoxReference("123", "i-0a61f580df1acfa65"), "0a61f580") { |
| 311 | t.Fatal("the reference shows the instance id") |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | // The console reads a group box's report under the reference it derives |
| 316 | // from the roster's pinned instance, never the label in the entry; every |
| 317 | // other box under its reference, exactly as before. |
| 318 | func TestPrivateBoxReportDerivesAGroupBoxsKey(t *testing.T) { |
| 319 | d, _, _ := fixture(t) |
| 320 | ctx := context.Background() |
| 321 | put := func(key, instance string) { |
| 322 | raw, _ := json.Marshal(status.Box{GeneratedAt: time.Now(), Metrics: &status.BoxMetrics{InstanceID: instance}}) |
| 323 | if _, err := d.Store.Put(ctx, key, raw, ""); err != nil { |
| 324 | t.Fatal(err) |
| 325 | } |
| 326 | } |
| 327 | derived := GroupBoxReference("123", "i-g") |
| 328 | put("status/boxes/"+derived+".json", "i-g") |
| 329 | put("status/boxes/chosen.json", "i-chosen") |
| 330 | put("status/boxes/annie-welding.json", "i-old") |
| 331 | r, err := d.PrivateBoxReport(ctx, "chosen", Box{Label: "chosen", Group: "routers-ie", Cloud: Cloud{Account: "123", InstanceID: "i-g"}}) |
| 332 | if err != nil || r.InstanceID != "i-g" { |
| 333 | t.Fatalf("group box: %+v %v", r, err) |
| 334 | } |
| 335 | r, err = d.PrivateBoxReport(ctx, "annie-welding", Box{Label: "annie-welding", Cloud: Cloud{Account: "123", InstanceID: "i-old"}}) |
| 336 | if err != nil || r.InstanceID != "i-old" { |
| 337 | t.Fatalf("a box an operator enrolled: %+v %v", r, err) |
| 338 | } |
| 339 | if _, err := d.PrivateBoxReport(ctx, derived, Box{Group: "routers-ie"}); !errors.Is(err, ErrRoster) { |
| 340 | t.Fatalf("a group box with no pinned instance: %v", err) |
| 341 | } |
| 342 | } |
| 343 | |
| 344 | // A group box is admitted only under the reference its instance derives: |
| 345 | // an attempt under a reference the box chose is refused before the cloud |
| 346 | // is asked, even for a member. |
| 347 | func TestEnrollAutomaticRefusesAReferenceTheBoxChose(t *testing.T) { |
| 348 | w := newMemberWorld(t, member("routers-ie")) |
| 349 | in := w.in |
| 350 | in.Reference, in.Label = "chosen", "chosen" |
| 351 | o, a, err := w.d.BeginEnrollmentAttempt(context.Background(), w.sg, w.o, in, w.raw, "i-new") |
| 352 | if err != nil { |
| 353 | t.Fatal(err) |
| 354 | } |
| 355 | in.AttemptID, in.ConfirmationID = a.AttemptID, a.ConfirmationID |
| 356 | if _, _, err := w.d.EnrollAutomatic(context.Background(), w.sg, o, w.v, in, w.raw, "i-new", w.cloud, w.now); !errors.Is(err, ErrRoster) || len(w.cloud.calls) != 0 { |
| 357 | t.Fatalf("a chosen reference: %v, calls %v", err, w.cloud.calls) |
| 358 | } |
| 359 | } |
| 360 | |
amadan