amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostfence
2
3import (
4 "context"
5 "crypto/ed25519"
6 "crypto/rand"
7 "encoding/json"
8 "github.com/aws/aws-sdk-go-v2/aws"
9 awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware"
10 "github.com/aws/aws-sdk-go-v2/service/ec2"
11 "github.com/aws/aws-sdk-go-v2/service/ec2/types"
12 "github.com/carlosframework/platform/internal/awsinv"
13 "github.com/carlosframework/platform/internal/boxvolumes"
14 "github.com/carlosframework/platform/internal/hostactors"
15 "github.com/carlosframework/platform/internal/hostfloor"
16 "github.com/carlosframework/platform/internal/release"
17 "strings"
18 "testing"
19 "time"
20)
21
22type terminatedEC2 struct {
23 state types.InstanceStateName
24 missing bool
25 owner string
26 after func()
27}
28
29func (f terminatedEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) {
30 out := &ec2.DescribeInstancesOutput{}
31 if !f.missing {
32 out.Reservations = []types.Reservation{{OwnerId: aws.String(f.owner), Instances: []types.Instance{{InstanceId: aws.String(in.InstanceIds[0]), State: &types.InstanceState{Name: f.state}}}}}
33 }
34 if f.after != nil {
35 f.after()
36 }
37 awsmiddleware.SetRequestIDMetadata(&out.ResultMetadata, "request")
38 return out, nil
39}
40func terminationFixture(t *testing.T) (TerminationWorker, release.Store) {
41 t.Helper()
42 ctx := context.Background()
43 pub, key, _ := ed25519.GenerateKey(rand.Reader)
44 s := release.NewDirStore(t.TempDir())
45 now := time.Now().UTC()
46 r := hostactors.Roster{V: 1, Revision: 1, DeploymentBucket: "bucket", Boxes: map[string]hostactors.Box{}, Consoles: map[string]hostactors.Console{}, Holds: map[string]hostactors.Reservation{}, Activations: map[string]hostactors.Reservation{}}
47 for i, ref := range []string{"worker", "dead"} {
48 en := strings.Repeat(string(rune('a'+i)), 32)
49 r.Boxes[ref] = hostactors.Box{Enrollment: en, Namespace: "platform", BoxID: ref, Label: ref, Region: "ie", Pool: "default", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + ref}, EvidenceSHA256: strings.Repeat("e", 64), State: "active"}
50 }
51 raw, _ := release.SignRootEnvelope(release.RootSigner(key), r)
52 if _, err := s.Put(ctx, hostactors.Key, raw, ""); err != nil {
53 t.Fatal(err)
54 }
55 return TerminationWorker{Fences: s, Enabled: true, Directory: hostactors.Directory{Store: s, PublicKey: pub, Bucket: "bucket"}, Self: Worker{Box: "worker", Enrollment: r.Boxes["worker"].Enrollment}, Now: func() time.Time { return now }}, s
56}
57func TestTerminationAloneFencesAndAbsenceNeverDoes(t *testing.T) {
58 for _, test := range []struct {
59 name string
60 state types.InstanceStateName
61 missing bool
62 owner string
63 ok bool
64 }{
65 {"terminated-no-volumes", types.InstanceStateNameTerminated, false, "123", true},
66 {"stopped", types.InstanceStateNameStopped, false, "123", false},
67 {"shutting-down", types.InstanceStateNameShuttingDown, false, "123", false},
68 {"absent", types.InstanceStateNameTerminated, true, "123", false},
69 {"wrong-account", types.InstanceStateNameTerminated, false, "456", false},
70 } {
71 t.Run(test.name, func(t *testing.T) {
72 w, s := terminationFixture(t)
73 w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) {
74 return terminatedEC2{state: test.state, missing: test.missing, owner: test.owner}, nil
75 }
76 ref, err := w.Describe(context.Background(), "dead")
77 if (err == nil) != test.ok {
78 t.Fatal(ref, err)
79 }
80 if test.ok {
81 raw, _, err := s.Get(context.Background(), ref.Key)
82 if err != nil {
83 t.Fatal(err)
84 }
85 var f Termination
86 _ = json.Unmarshal(raw, &f)
87 old, err := time.Parse(time.RFC3339Nano, f.ObservedAt)
88 if err != nil {
89 t.Fatal(err)
90 }
91 w.Now = func() time.Time { return old.Add(time.Hour) }
92 again, err := w.Describe(context.Background(), "dead")
93 if err != nil || again != ref {
94 t.Fatal("retry moved evidence", again, err)
95 }
96 } else {
97 keys, _ := s.List(context.Background(), "control/box-fences/")
98 if len(keys) != 0 {
99 t.Fatal("refusal wrote a fence")
100 }
101 }
102 })
103 }
104}
105func TestDestroyedLocalIsIndependentAndNeedsHistory(t *testing.T) {
106 now := time.Now().UTC()
107 enrolledAt := now.Add(-time.Minute)
108 e := boxvolumes.Entry{Box: "dead", Enrollment: "enrollment", Cloud: hostactors.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-dead"}}
109 if boxvolumes.Discharge(nil, e, enrolledAt, now).Discharge {
110 t.Fatal("empty terminated mappings proved destroyed storage")
111 }
112 r := boxvolumes.Record{V: 1, Box: e.Box, Enrollment: e.Enrollment, Cloud: e.Cloud, InstanceState: "running", ObservedAt: enrolledAt.Format(time.RFC3339Nano), RequestID: "request", BlockDevices: []boxvolumes.Device{{Device: "root", VolumeID: "volume", DeleteOnTermination: true, Status: "attached"}}}
113 if !boxvolumes.Discharge([]boxvolumes.Record{r}, e, enrolledAt, now).Discharge {
114 t.Fatal("complete true history refused")
115 }
116 r.BlockDevices[0].DeleteOnTermination = false
117 if boxvolumes.Discharge([]boxvolumes.Record{r}, e, enrolledAt, now).Discharge {
118 t.Fatal("surviving volume discharged")
119 }
120}
121
122func TestFencedWorkerCannotProduceNewEvidence(t *testing.T) {
123 w, s := terminationFixture(t)
124 o, err := w.Directory.Read(context.Background())
125 if err != nil {
126 t.Fatal(err)
127 }
128 self := o.Roster.Boxes["worker"]
129 dead := o.Roster.Boxes["dead"]
130 fence := Termination{V: 1, Box: "worker", Enrollment: self.Enrollment, Cloud: self.Cloud, State: "terminated", ObservedAt: w.Now().Format(time.RFC3339Nano), Via: "describe", RequestID: "request", WrittenBy: Worker{Box: "dead", Enrollment: dead.Enrollment}}
131 raw, _ := json.Marshal(fence)
132 if _, err := s.Put(context.Background(), FenceKey(self.Enrollment), raw, ""); err != nil {
133 t.Fatal(err)
134 }
135 w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) {
136 return terminatedEC2{state: types.InstanceStateNameTerminated, owner: "123"}, nil
137 }
138 if _, err := w.Describe(context.Background(), "dead"); err == nil {
139 t.Fatal("fenced worker produced new execution authority")
140 }
141 keys, err := s.List(context.Background(), "control/box-fences/")
142 if err != nil || len(keys) != 1 {
143 t.Fatal("refused worker wrote another fence", keys, err)
144 }
145}
146
147func TestTerminationReferenceIncludesExactEnrollmentAndBytes(t *testing.T) {
148 w, s := terminationFixture(t)
149 w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) {
150 return terminatedEC2{state: types.InstanceStateNameTerminated, owner: "123"}, nil
151 }
152 ref, err := w.Describe(context.Background(), "dead")
153 if err != nil {
154 t.Fatal(err)
155 }
156 raw, _, err := s.Get(context.Background(), ref.Key)
157 if err != nil {
158 t.Fatal(err)
159 }
160 var fence Termination
161 if json.Unmarshal(raw, &fence) != nil || ref.Enrollment != fence.Enrollment || ref.Key != FenceKey(fence.Enrollment) || ref.SHA256 != hostfloor.Digest(raw) {
162 t.Fatal("reference is not bound to the verified enrollment and bytes", ref)
163 }
164 wire, err := json.Marshal(ref)
165 if err != nil {
166 t.Fatal(err)
167 }
168 var fields map[string]string
169 if json.Unmarshal(wire, &fields) != nil || len(fields) != 3 || fields["enrollment"] != fence.Enrollment || fields["key"] != ref.Key || fields["sha256"] != ref.SHA256 {
170 t.Fatal("journal reference lacks the adopted D2 wire contract", string(wire))
171 }
172}
173
174func TestTerminationFenceTimeIsAfterTheVerifiedRead(t *testing.T) {
175 w, s := terminationFixture(t)
176 start := w.Now()
177 now := start
178 w.Now = func() time.Time { return now }
179 w.Reader = func(context.Context, string) (awsinv.ExactEC2API, error) {
180 return terminatedEC2{state: types.InstanceStateNameTerminated, owner: "123", after: func() { now = start.Add(time.Minute) }}, nil
181 }
182 ref, err := w.Describe(context.Background(), "dead")
183 if err != nil {
184 t.Fatal(err)
185 }
186 raw, _, err := s.Get(context.Background(), ref.Key)
187 if err != nil {
188 t.Fatal(err)
189 }
190 var fence Termination
191 if err = json.Unmarshal(raw, &fence); err != nil {
192 t.Fatal(err)
193 }
194 if fence.ObservedAt != now.Format(time.RFC3339Nano) {
195 t.Fatal("fence time predates completion of the terminal-state read", fence.ObservedAt, now)
196 }
197}
198