| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/x509" |
| 7 | "encoding/base64" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "flag" |
| 11 | "fmt" |
| 12 | "io" |
| 13 | "net/http" |
| 14 | "os" |
| 15 | "strings" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/boxid" |
| 19 | "github.com/carlosframework/platform/internal/cliflag" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | ) |
| 22 | |
| 23 | // carlos system box publish-record: one publishBoxRecord attempt, by hand, |
| 24 | // without the CARLOS_REGION_HA gate boxRecordLoop has (census ruling |
| 25 | // box-publish-record, R1-R4). |
| 26 | // |
| 27 | // The host claim's enrollment needs a box's signed identity record at |
| 28 | // control/boxes/<box>.json, and the agent writes it only when region HA is |
| 29 | // on. This verb lets an operator publish it once, with the box's own AWS |
| 30 | // credentials and IMDS: AWS signs the identity document, not the box and |
| 31 | // not the operator. |
| 32 | // |
| 33 | // It runs as the agent user and refuses root before any store, IMDS or |
| 34 | // host read (R1); the one supported way to run it is systemd-run with |
| 35 | // User=carlos and host.env as its EnvironmentFile (R2). It refuses on a |
| 36 | // fleet box, as the loop skips one, and anywhere without an S3 deployment |
| 37 | // bucket: only S3's If-None-Match makes the write create-only, and a |
| 38 | // deployment directory's check-then-rename does not. |
| 39 | // |
| 40 | // It reads the record already under the box's key first, and keeps one |
| 41 | // that enrollment's own verifier accepts as evidence for this box |
| 42 | // (heldEvidence: boxid.Verifier.Enroll, with the account, AWS region and |
| 43 | // instance id taken from this box's current IMDS identity). Its bytes are |
| 44 | // never touched, however old: an instance's identity document changes |
| 45 | // with its lifecycle (pendingTime, instanceType) while the instance stays |
| 46 | // the same, and the record cannot be replaced. Anything the verifier |
| 47 | // refuses is refused and left alone. Keeping a record says only that |
| 48 | // valid evidence is in place; enrollment still makes its roster and |
| 49 | // operator checks. With nothing there, it calls publishBoxRecord through |
| 50 | // boxRecordStore, which can read and create this box's one record key and |
| 51 | // nothing else (R4), so it can never delete or overwrite: the dev bucket's |
| 52 | // box_records policy (BoxRecordsAreCreateOnly, BoxRecordsAreNeverDeleted) |
| 53 | // denies both anyway. |
| 54 | |
| 55 | const boxPublishRecordCommand = "systemd-run --wait --pipe -p EnvironmentFile=/etc/carlos/host.env -p User=carlos /opt/carlos/carlos system box publish-record" |
| 56 | |
| 57 | const boxPublishRecordUsage = "usage: carlos system box publish-record\n\n" + |
| 58 | "Publish this box's AWS-signed identity record to the deployment bucket, once. It never replaces or deletes a record.\n" + |
| 59 | "Run it on the box as the agent user, with host.env loaded:\n\n" + |
| 60 | " " + boxPublishRecordCommand + "\n" |
| 61 | |
| 62 | // errBoxRecordStore is boxRecordStore refusing anything but reading and |
| 63 | // creating this box's own record. publishBoxRecord never asks for more, so |
| 64 | // a user only sees it if that changes. |
| 65 | var errBoxRecordStore = errors.New("publish-record only reads and creates this box's own record") |
| 66 | |
| 67 | // boxRecordStore is the only store publish-record gives publishBoxRecord |
| 68 | // (R4): Get on this box's record key, and Put on it create-only (ifMatch |
| 69 | // "", which S3Store sends as If-None-Match: *). Everything else is |
| 70 | // refused before it reaches the bucket. attempted is whether a create |
| 71 | // reached the bucket, and wrote whether one was answered as done. |
| 72 | type boxRecordStore struct { |
| 73 | inner release.Store |
| 74 | key string |
| 75 | attempted bool |
| 76 | wrote bool |
| 77 | } |
| 78 | |
| 79 | func (s *boxRecordStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 80 | if key != s.key { |
| 81 | return nil, "", errBoxRecordStore |
| 82 | } |
| 83 | return s.inner.Get(ctx, key) |
| 84 | } |
| 85 | |
| 86 | func (s *boxRecordStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| 87 | if key != s.key || ifMatch != "" { |
| 88 | return "", errBoxRecordStore |
| 89 | } |
| 90 | s.attempted = true |
| 91 | etag, err := s.inner.Put(ctx, key, body, "") |
| 92 | if err == nil { |
| 93 | s.wrote = true |
| 94 | } |
| 95 | return etag, err |
| 96 | } |
| 97 | |
| 98 | func (s *boxRecordStore) List(context.Context, string) ([]string, error) { |
| 99 | return nil, errBoxRecordStore |
| 100 | } |
| 101 | |
| 102 | func (s *boxRecordStore) ListDirs(context.Context, string) ([]string, error) { |
| 103 | return nil, errBoxRecordStore |
| 104 | } |
| 105 | |
| 106 | func (s *boxRecordStore) Delete(context.Context, string) error { return errBoxRecordStore } |
| 107 | |
| 108 | // The lines publish-record prints. Every one is in the copy review deck. |
| 109 | func boxRecordPublishedLine(box, instance string) string { |
| 110 | return fmt.Sprintf("published the identity record for %s (instance %s) to %s", box, instance, boxid.Key(box)) |
| 111 | } |
| 112 | |
| 113 | func boxRecordInPlaceLine(box, instance string) string { |
| 114 | return fmt.Sprintf("the record at %s is valid AWS-signed evidence for %s (instance %s), so nothing was written. enrollment still makes its own checks before it accepts the box", boxid.Key(box), box, instance) |
| 115 | } |
| 116 | |
| 117 | const ( |
| 118 | boxRecordRootRefusal = "carlos system box publish-record runs as the agent user, not root. run it with systemd-run -p User=carlos." |
| 119 | boxRecordFleetRefusal = "this is a fleet box. fleet boxes publish no identity record, so nothing was written" |
| 120 | boxRecordNoBucket = "this box has no deployment bucket (CARLOS_DEPLOYMENT_BUCKET is not set). the record is written only to an S3 bucket, which refuses to overwrite it, so nothing was written" |
| 121 | boxRecordNoNameRefusal = "this box's name could not be read, so nothing was written" |
| 122 | ) |
| 123 | |
| 124 | // errOwnIdentityIncomplete is the IMDS identity document naming no account, |
| 125 | // region or instance: there is nothing to check a stored record against, |
| 126 | // and nothing worth publishing. |
| 127 | var errOwnIdentityIncomplete = errors.New("the identity document AWS served this box names no account, region or instance") |
| 128 | |
| 129 | func boxRecordBadName(box string) error { |
| 130 | return fmt.Errorf("this box's name %q is not a valid box reference, so nothing was written", box) |
| 131 | } |
| 132 | |
| 133 | func boxRecordStoreFailure(err error) error { |
| 134 | return fmt.Errorf("the deployment bucket could not be opened: %v. nothing was written", err) |
| 135 | } |
| 136 | |
| 137 | func boxRecordRefusal(err error) error { |
| 138 | return fmt.Errorf("refused: %v. nothing was changed", err) |
| 139 | } |
| 140 | |
| 141 | func boxRecordMismatch(box, what string) error { |
| 142 | return fmt.Errorf("refused: the record already under %s is not valid evidence for this box: %s. nothing was changed", box, what) |
| 143 | } |
| 144 | |
| 145 | // boxRecordRetryable is a failure before any write reached the bucket. |
| 146 | func boxRecordRetryable(box string, err error) error { |
| 147 | return fmt.Errorf("the identity record for %s was not published: %v. nothing was written, so it is safe to run again", box, err) |
| 148 | } |
| 149 | |
| 150 | // boxRecordUncertain is a failure once a create reached the bucket: S3 may |
| 151 | // have committed it. A second run only ever creates, and reports what it |
| 152 | // finds. |
| 153 | func boxRecordUncertain(box string, err error) error { |
| 154 | return fmt.Errorf("the identity record for %s may or may not have been written: %v. it is safe to run again, which only ever creates the record and says what is there", box, err) |
| 155 | } |
| 156 | |
| 157 | // awsIdentity is the part of an identity document enrollment checks. |
| 158 | type awsIdentity struct { |
| 159 | Account string `json:"accountId"` |
| 160 | Region string `json:"region"` |
| 161 | Instance string `json:"instanceId"` |
| 162 | } |
| 163 | |
| 164 | // identityOf reads rec's identity document. It does not check the |
| 165 | // signature. |
| 166 | func identityOf(rec boxid.Record) (awsIdentity, bool) { |
| 167 | doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument) |
| 168 | if err != nil { |
| 169 | return awsIdentity{}, false |
| 170 | } |
| 171 | var id awsIdentity |
| 172 | if json.Unmarshal(doc, &id) != nil { |
| 173 | return awsIdentity{}, false |
| 174 | } |
| 175 | return id, true |
| 176 | } |
| 177 | |
| 178 | // heldEvidence says why raw, the record already under the box's key, is |
| 179 | // not evidence for this box, or "" when it is. The verdict is |
| 180 | // boxid.Verifier.Enroll's, with the account, AWS region and instance id |
| 181 | // (TofuInstanceID) taken from own, this box's current IMDS identity, and |
| 182 | // certs as enrollment's: a strict v1 envelope naming this box, no other |
| 183 | // fields and nothing after it, and AWS's signature under the certificate |
| 184 | // for the document's region. There is no freshness check: the document's |
| 185 | // mutable fields (pendingTime, instanceType) and written_at may differ |
| 186 | // from today's. The rest only names the refusal, in Enroll's own order. |
| 187 | func heldEvidence(raw []byte, box string, own awsIdentity, certs map[string]*x509.Certificate) string { |
| 188 | v := boxid.Verifier{Account: own.Account, Certs: certs} |
| 189 | _, err := v.Enroll(raw, boxid.Expect{Box: box, AWSRegion: own.Region, TofuInstanceID: own.Instance}) |
| 190 | if err == nil { |
| 191 | return "" |
| 192 | } |
| 193 | const unreadable = "it cannot be read as an identity record" |
| 194 | dec := json.NewDecoder(bytes.NewReader(raw)) |
| 195 | dec.DisallowUnknownFields() |
| 196 | var held boxid.Record |
| 197 | if dec.Decode(&held) != nil { |
| 198 | return unreadable |
| 199 | } |
| 200 | if _, terr := dec.Token(); !errors.Is(terr, io.EOF) { |
| 201 | return unreadable |
| 202 | } |
| 203 | switch { |
| 204 | case held.V != 1: |
| 205 | return fmt.Sprintf("its version is %d, not 1", held.V) |
| 206 | case held.Box != box: |
| 207 | return fmt.Sprintf("it is for box %q", held.Box) |
| 208 | } |
| 209 | id, ok := identityOf(held) |
| 210 | switch { |
| 211 | case !ok: |
| 212 | return unreadable |
| 213 | case id.Instance != own.Instance: |
| 214 | return fmt.Sprintf("it is signed for instance %s, and this box is %s", id.Instance, own.Instance) |
| 215 | case id.Account != own.Account: |
| 216 | return fmt.Sprintf("it is signed for account %s, and this box is in account %s", id.Account, own.Account) |
| 217 | case id.Region != own.Region: |
| 218 | return fmt.Sprintf("it is signed in region %s, and this box is in %s", id.Region, own.Region) |
| 219 | } |
| 220 | reason := strings.TrimPrefix(err.Error(), boxid.ErrUnverified.Error()+": ") |
| 221 | return fmt.Sprintf("it does not verify as AWS-signed evidence: %s", reason) |
| 222 | } |
| 223 | |
| 224 | // boxPublishRun is one run of the verb, with its seams: the environment, |
| 225 | // the effective uid (R1), the machine id (a host read, made only after the |
| 226 | // uid check), the deployment store, the IMDS read and the AWS signing |
| 227 | // certificates enrollment verifies with. |
| 228 | type boxPublishRun struct { |
| 229 | getenv func(string) string |
| 230 | geteuid func() int |
| 231 | machineID func() string |
| 232 | open func(context.Context, func(string) string) (release.Store, error) |
| 233 | read func(box string) (boxid.Record, error) |
| 234 | certs func() (map[string]*x509.Certificate, error) |
| 235 | } |
| 236 | |
| 237 | // publish is the verb. It returns the line to print, or the refusal. |
| 238 | func (r boxPublishRun) publish(ctx context.Context) (string, error) { |
| 239 | if r.geteuid() == 0 { |
| 240 | return "", errors.New(boxRecordRootRefusal) |
| 241 | } |
| 242 | if shouldSkipPublisherElection(r.getenv) { |
| 243 | return "", errors.New(boxRecordFleetRefusal) |
| 244 | } |
| 245 | if r.getenv("CARLOS_DEPLOYMENT_BUCKET") == "" { |
| 246 | return "", errors.New(boxRecordNoBucket) |
| 247 | } |
| 248 | box := metricsBoxLabel(r.getenv, r.machineID()) |
| 249 | if box == "" { |
| 250 | return "", errors.New(boxRecordNoNameRefusal) |
| 251 | } |
| 252 | if !boxid.ValidBox(box) { |
| 253 | return "", boxRecordBadName(box) |
| 254 | } |
| 255 | opened, err := r.open(ctx, r.getenv) |
| 256 | if err != nil { |
| 257 | return "", boxRecordStoreFailure(err) |
| 258 | } |
| 259 | if _, ok := opened.(*release.S3Store); !ok { |
| 260 | return "", errors.New(boxRecordNoBucket) |
| 261 | } |
| 262 | store := &boxRecordStore{inner: opened, key: boxid.Key(box)} |
| 263 | |
| 264 | existing, _, gerr := store.Get(ctx, store.key) |
| 265 | if gerr != nil && !errors.Is(gerr, os.ErrNotExist) { |
| 266 | return "", boxRecordRetryable(box, gerr) |
| 267 | } |
| 268 | mine, err := r.read(box) |
| 269 | if err != nil { |
| 270 | return "", boxRecordRetryable(box, err) |
| 271 | } |
| 272 | own, ok := identityOf(mine) |
| 273 | if !ok || own.Account == "" || own.Region == "" || own.Instance == "" { |
| 274 | return "", boxRecordRetryable(box, errOwnIdentityIncomplete) |
| 275 | } |
| 276 | // keep decides a record that is already there: kept as it is when the |
| 277 | // verifier accepts it, refused otherwise. |
| 278 | keep := func(raw []byte) (string, error) { |
| 279 | certs, err := r.certs() |
| 280 | if err != nil { |
| 281 | return "", boxRecordRetryable(box, err) |
| 282 | } |
| 283 | if what := heldEvidence(raw, box, own, certs); what != "" { |
| 284 | return "", boxRecordMismatch(box, what) |
| 285 | } |
| 286 | return boxRecordInPlaceLine(box, own.Instance), nil |
| 287 | } |
| 288 | if gerr == nil { |
| 289 | return keep(existing) |
| 290 | } |
| 291 | |
| 292 | done, err := publishBoxRecord(ctx, store, box, func() (boxid.Record, error) { return mine, nil }) |
| 293 | switch { |
| 294 | case err != nil && done: |
| 295 | return "", boxRecordRefusal(err) |
| 296 | case err != nil && store.attempted: |
| 297 | return "", boxRecordUncertain(box, err) |
| 298 | case err != nil: |
| 299 | return "", boxRecordRetryable(box, err) |
| 300 | case store.attempted: |
| 301 | // Either this create landed, or a retry of it was answered 412 |
| 302 | // and boxid.Publish found exactly these bytes, written_at |
| 303 | // included: this run's record either way. |
| 304 | return boxRecordPublishedLine(box, own.Instance), nil |
| 305 | } |
| 306 | // No create was attempted: a record arrived after the read above and |
| 307 | // publishBoxRecord took it as this instance's. It is held to the same |
| 308 | // verifier as one that was there from the start. |
| 309 | raw, _, err := store.Get(ctx, store.key) |
| 310 | if err != nil { |
| 311 | return "", boxRecordRetryable(box, err) |
| 312 | } |
| 313 | return keep(raw) |
| 314 | } |
| 315 | |
| 316 | // RunSystemBoxPublishRecord is `carlos system box publish-record`. |
| 317 | func RunSystemBoxPublishRecord(args []string) { |
| 318 | fs := flag.NewFlagSet("system box publish-record", flag.ExitOnError) |
| 319 | fs.Usage = cliflag.Usage(fs, boxPublishRecordUsage) |
| 320 | _ = fs.Parse(args) |
| 321 | if fs.NArg() != 0 { |
| 322 | fs.Usage() |
| 323 | os.Exit(2) |
| 324 | } |
| 325 | client := &http.Client{Timeout: 2 * time.Second} |
| 326 | run := boxPublishRun{getenv: os.Getenv, geteuid: os.Geteuid, machineID: readMachineID, open: openDeployStoreEnv, |
| 327 | read: func(box string) (boxid.Record, error) { return readIdentityRecord(client, imdsBase, box, time.Now()) }, |
| 328 | certs: boxid.EmbeddedCerts} |
| 329 | line, err := run.publish(context.Background()) |
| 330 | if err != nil { |
| 331 | fmt.Fprintln(os.Stderr, err) |
| 332 | os.Exit(1) |
| 333 | } |
| 334 | fmt.Println(line) |
| 335 | } |
| 336 | |
amadan