| 1 | // Package termfence decides whether an observation proves that a box's |
| 2 | // machine has been terminated (region HA spec 1 §5.9, invariants HA7 and |
| 3 | // H19). |
| 4 | // |
| 5 | // Only the state "terminated" counts, only for the instance, account and |
| 6 | // region the box's roster entry records under cloud, and only when the |
| 7 | // worker that read it is in that same region. Silence is never evidence, |
| 8 | // and no disk condition applies: volume records decide only whether the |
| 9 | // dead box's local chores are discharged (internal/boxvolumes). |
| 10 | // |
| 11 | // This package reads and decides. Writing the fence record and citing it |
| 12 | // in a claim CAS wait for the claim's acceptance of L2 to L5. |
| 13 | package termfence |
| 14 | |
| 15 | import ( |
| 16 | "context" |
| 17 | "encoding/json" |
| 18 | "errors" |
| 19 | "fmt" |
| 20 | "time" |
| 21 | |
| 22 | "github.com/aws/aws-sdk-go-v2/aws" |
| 23 | awsmiddleware "github.com/aws/aws-sdk-go-v2/aws/middleware" |
| 24 | "github.com/aws/aws-sdk-go-v2/service/ec2" |
| 25 | "github.com/aws/smithy-go" |
| 26 | |
| 27 | "github.com/carlosframework/platform/internal/boxid" |
| 28 | ) |
| 29 | |
| 30 | const ( |
| 31 | ViaDescribe = "describe" |
| 32 | ViaEvent = "event" |
| 33 | |
| 34 | stateTerminated = "terminated" |
| 35 | eventSource = "aws.ec2" |
| 36 | eventDetailType = "EC2 Instance State-change Notification" |
| 37 | ) |
| 38 | |
| 39 | // ErrNotEvidence wraps every reason an observation does not fence a box. |
| 40 | var ErrNotEvidence = errors.New("termfence: not evidence of termination") |
| 41 | |
| 42 | // Observation is what one source said about one instance. Found is false |
| 43 | // when the source said nothing about it: an empty reply, another instance. |
| 44 | type Observation struct { |
| 45 | Via string |
| 46 | Found bool |
| 47 | InstanceID string |
| 48 | Account string |
| 49 | Region string |
| 50 | State string |
| 51 | EventID string |
| 52 | RequestID string |
| 53 | } |
| 54 | |
| 55 | // Box is a roster entry as the fence needs it: the reference, the |
| 56 | // enrollment, and the cloud identity enrollment proved (boxid.Enroll). |
| 57 | type Box struct { |
| 58 | Ref string |
| 59 | Enrollment string |
| 60 | Cloud boxid.Cloud |
| 61 | } |
| 62 | |
| 63 | func (b Box) complete() bool { return b.Ref != "" && b.Enrollment != "" && b.Cloud.Complete() } |
| 64 | |
| 65 | // WrittenBy names the regional worker that made the read. |
| 66 | type WrittenBy struct { |
| 67 | Box string `json:"box"` |
| 68 | Enrollment string `json:"enrollment"` |
| 69 | } |
| 70 | |
| 71 | // Record is §5.9's fence record. Its key (control/box-fences/<enrollment>.json) |
| 72 | // and its writer belong to the claim's L4 and are not built here. |
| 73 | type Record struct { |
| 74 | V int `json:"v"` |
| 75 | Box string `json:"box"` |
| 76 | Enrollment string `json:"enrollment"` |
| 77 | Cloud boxid.Cloud `json:"cloud"` |
| 78 | State string `json:"state"` |
| 79 | ObservedAt string `json:"observed_at"` |
| 80 | Via string `json:"via"` |
| 81 | RequestID string `json:"request_id,omitempty"` |
| 82 | EventID string `json:"event_id,omitempty"` |
| 83 | WrittenBy WrittenBy `json:"written_by"` |
| 84 | } |
| 85 | |
| 86 | // FromDescribe reads one instance out of a DescribeInstances reply. region |
| 87 | // is the region the call was made in: the reply does not carry it. A reply |
| 88 | // that does not mention instanceID yields Found false, which Verify |
| 89 | // refuses: AWS forgets a terminated instance within about an hour, so |
| 90 | // absence proves nothing. |
| 91 | func FromDescribe(out *ec2.DescribeInstancesOutput, instanceID, region string) Observation { |
| 92 | o := Observation{Via: ViaDescribe, Region: region} |
| 93 | if out == nil || instanceID == "" { |
| 94 | return o |
| 95 | } |
| 96 | if id, ok := awsmiddleware.GetRequestIDMetadata(out.ResultMetadata); ok { |
| 97 | o.RequestID = id |
| 98 | } |
| 99 | for _, r := range out.Reservations { |
| 100 | for _, inst := range r.Instances { |
| 101 | if aws.ToString(inst.InstanceId) != instanceID { |
| 102 | continue |
| 103 | } |
| 104 | o.Found, o.InstanceID, o.Account = true, instanceID, aws.ToString(r.OwnerId) |
| 105 | if inst.State != nil { |
| 106 | o.State = string(inst.State.Name) |
| 107 | } |
| 108 | return o |
| 109 | } |
| 110 | } |
| 111 | return o |
| 112 | } |
| 113 | |
| 114 | // FromEvent reads an EventBridge "EC2 Instance State-change Notification". |
| 115 | // It refuses anything else: the queue's policy admits only the rule that |
| 116 | // forwards these, and this is the second check. |
| 117 | func FromEvent(body []byte) (Observation, error) { |
| 118 | var ev struct { |
| 119 | ID string `json:"id"` |
| 120 | DetailType string `json:"detail-type"` |
| 121 | Source string `json:"source"` |
| 122 | Account string `json:"account"` |
| 123 | Region string `json:"region"` |
| 124 | Detail struct { |
| 125 | InstanceID string `json:"instance-id"` |
| 126 | State string `json:"state"` |
| 127 | } `json:"detail"` |
| 128 | } |
| 129 | if err := json.Unmarshal(body, &ev); err != nil { |
| 130 | return Observation{}, fmt.Errorf("termfence: event is not JSON: %w", err) |
| 131 | } |
| 132 | switch { |
| 133 | case ev.Source != eventSource: |
| 134 | return Observation{}, fmt.Errorf("termfence: event source %q is not %s", ev.Source, eventSource) |
| 135 | case ev.DetailType != eventDetailType: |
| 136 | return Observation{}, fmt.Errorf("termfence: event detail-type %q is not a state change", ev.DetailType) |
| 137 | case ev.ID == "": |
| 138 | return Observation{}, errors.New("termfence: event has no id") |
| 139 | } |
| 140 | return Observation{Via: ViaEvent, Found: ev.Detail.InstanceID != "", InstanceID: ev.Detail.InstanceID, |
| 141 | Account: ev.Account, Region: ev.Region, State: ev.Detail.State, EventID: ev.ID}, nil |
| 142 | } |
| 143 | |
| 144 | // EC2 is the one call this package makes. *ec2.Client satisfies it; tests |
| 145 | // use a fake. It is a read: the box role holds no EC2 write right. |
| 146 | type EC2 interface { |
| 147 | DescribeInstances(ctx context.Context, in *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) |
| 148 | } |
| 149 | |
| 150 | // DescribeInstance asks EC2 about c.InstanceID alone, in c.Region whatever |
| 151 | // region the client was built for. InvalidInstanceID.NotFound comes back |
| 152 | // as an empty reply, which FromDescribe turns into Found false: absence, |
| 153 | // never termination. Every other error is returned. |
| 154 | func DescribeInstance(ctx context.Context, api EC2, c boxid.Cloud) (*ec2.DescribeInstancesOutput, error) { |
| 155 | if !c.Complete() { |
| 156 | return nil, errors.New("termfence: describe of an incomplete cloud identity") |
| 157 | } |
| 158 | region := c.Region |
| 159 | out, err := api.DescribeInstances(ctx, &ec2.DescribeInstancesInput{InstanceIds: []string{c.InstanceID}}, |
| 160 | func(o *ec2.Options) { o.Region = region }) |
| 161 | var apiErr smithy.APIError |
| 162 | if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidInstanceID.NotFound" { |
| 163 | return &ec2.DescribeInstancesOutput{}, nil |
| 164 | } |
| 165 | return out, err |
| 166 | } |
| 167 | |
| 168 | func refuse(format string, a ...any) error { |
| 169 | return fmt.Errorf("%w: %s", ErrNotEvidence, fmt.Sprintf(format, a...)) |
| 170 | } |
| 171 | |
| 172 | // Verify returns box's fence record when o proves that the instance its |
| 173 | // roster entry records is terminated, read by worker, a regional worker of |
| 174 | // the same AWS region (spec §6: the fence refuses any observation not made |
| 175 | // in the box's own region). Whether worker is active in the roster is the |
| 176 | // caller's check, made before it reads. |
| 177 | func Verify(box, worker Box, o Observation, now time.Time) (Record, error) { |
| 178 | c := box.Cloud |
| 179 | switch { |
| 180 | case !box.complete(): |
| 181 | return Record{}, refuse("the box has no proven cloud identity") |
| 182 | case !worker.complete(): |
| 183 | return Record{}, refuse("the worker has no proven cloud identity") |
| 184 | case worker.Cloud.Region != c.Region: |
| 185 | return Record{}, refuse("worker %s is in %q, the box in %q", worker.Ref, worker.Cloud.Region, c.Region) |
| 186 | case o.Via != ViaDescribe && o.Via != ViaEvent: |
| 187 | return Record{}, refuse("source %q is not one this binary accepts", o.Via) |
| 188 | case !o.Found: |
| 189 | return Record{}, refuse("the source said nothing about %s; absence is not termination", c.InstanceID) |
| 190 | case o.InstanceID != c.InstanceID: |
| 191 | return Record{}, refuse("observed %s, the box is %s", o.InstanceID, c.InstanceID) |
| 192 | case o.Account != c.Account: |
| 193 | return Record{}, refuse("observed in account %q, the box is in %q", o.Account, c.Account) |
| 194 | case o.Region != c.Region: |
| 195 | return Record{}, refuse("observed in region %q, the box is in %q", o.Region, c.Region) |
| 196 | case o.State != stateTerminated: |
| 197 | return Record{}, refuse("state is %q; only %q counts", o.State, stateTerminated) |
| 198 | } |
| 199 | return Record{V: 1, Box: box.Ref, Enrollment: box.Enrollment, Cloud: c, State: o.State, |
| 200 | ObservedAt: now.UTC().Format(time.RFC3339), Via: o.Via, RequestID: o.RequestID, EventID: o.EventID, |
| 201 | WrittenBy: WrittenBy{Box: worker.Ref, Enrollment: worker.Enrollment}}, nil |
| 202 | } |
| 203 | |
amadan