| 1 | package hostactors |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/x509" |
| 6 | "encoding/base64" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "slices" |
| 12 | "sort" |
| 13 | "strings" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/boxid" |
| 17 | "github.com/carlosframework/platform/internal/hostfloor" |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | ) |
| 20 | |
| 21 | // The console's automatic enrollment pass (region HA spec 2 §5 step 5, |
| 22 | // A2-1, H16, H28). A box its region's group launched publishes its signed |
| 23 | // identity record (control/boxes/<ref>.json) and joins. For each such box |
| 24 | // not yet in the roster, the pass takes the next step its state allows: |
| 25 | // |
| 26 | // 1. no invitation: begin an enrollment attempt (BeginEnrollmentAttempt), |
| 27 | // then, on the next pass, confirm the instance's group membership with |
| 28 | // the console's own read (checkConfirmed, Members) and invite the box |
| 29 | // (Invite) while that read is under a minute old; |
| 30 | // 2. invited: wait for the box to arm its guard and publish its |
| 31 | // guard-ready report (VerifyReport, the same check membership.go makes |
| 32 | // when the box is admitted); |
| 33 | // 3. guard-ready: enroll it (EnrollAutomatic), which reads membership |
| 34 | // again, itself, immediately before the enrolling write. |
| 35 | // |
| 36 | // A read that fails is unknown: the pass asks again next time, with the |
| 37 | // same attempt, and its wait does not grow. Only the cloud's positive |
| 38 | // answer that the instance is not a current member (ErrNotMember), or a |
| 39 | // read that was over a minute old at its write (ErrMembershipStale), |
| 40 | // counts against the attempt: the pass then waits Patience before it |
| 41 | // begins another, doubling each time, so the record of a box the group |
| 42 | // has already replaced stops costing roster writes and describe calls. |
| 43 | // The invitation never changes. |
| 44 | // |
| 45 | // The box takes its group's role in the enrolling write (enroll.go). Only |
| 46 | // a deployment whose host floor is published is enrolled automatically: |
| 47 | // a box can arm its guard only under one. |
| 48 | // |
| 49 | // Nothing here names a provider beyond the identity the roster already |
| 50 | // rests on (H26, §25): the record is AWS's signed instance document, as |
| 51 | // for every enrolled box on main, and Members is an interface. |
| 52 | |
| 53 | // AutoEnroller is the pass's configuration and its memory of attempts. |
| 54 | type AutoEnroller struct { |
| 55 | Directory Directory |
| 56 | Signer release.Signer |
| 57 | // Evidence reads boxes' reports and release identities (the console's |
| 58 | // ExactEvidence). |
| 59 | Evidence Evidence |
| 60 | // Members is the console's own read of the cloud's group membership |
| 61 | // (awsmembership in production). |
| 62 | Members MembershipReader |
| 63 | // Certs verify identity records (boxid.EmbeddedCerts in production). |
| 64 | Certs map[string]*x509.Certificate |
| 65 | // Namespace and Pool are what a box is enrolled into: the console's |
| 66 | // own namespace, and the default pool. |
| 67 | Namespace, Pool string |
| 68 | Now func() time.Time |
| 69 | // Patience is how long the pass waits after an attempt is refused |
| 70 | // before it begins another. Each refused attempt doubles the wait, up |
| 71 | // to 32 times Patience, so a record whose instance is gone (a box the |
| 72 | // group replaced while it was joining) costs the roster a write an |
| 73 | // hour, not one a minute. An invitation resets it. |
| 74 | Patience time.Duration |
| 75 | // Logf records each membership read's request ids (C2's audit: the |
| 76 | // roster has no field for them). nil is silent. |
| 77 | Logf func(format string, args ...any) |
| 78 | |
| 79 | attempts map[string]attemptSeen |
| 80 | } |
| 81 | |
| 82 | // attemptSeen is what the pass remembers of a box's current attempt: its |
| 83 | // id, when the pass first saw it, how many attempts it has begun for the |
| 84 | // box since the last invitation, and why the cloud refused the current |
| 85 | // one ("" while it has not). Memory only: a console restart starts every |
| 86 | // box afresh. |
| 87 | type attemptSeen struct { |
| 88 | id string |
| 89 | since time.Time |
| 90 | tries int |
| 91 | refused string |
| 92 | } |
| 93 | |
| 94 | // wait is how long a refused attempt is kept after tries attempts. |
| 95 | func (e *AutoEnroller) wait(tries int) time.Duration { |
| 96 | return e.Patience << min(max(tries-1, 0), 5) |
| 97 | } |
| 98 | |
| 99 | // EnrollStep is what one pass did, or is waiting for, for one box. |
| 100 | type EnrollStep struct { |
| 101 | Box string |
| 102 | Step string |
| 103 | } |
| 104 | |
| 105 | // ErrNoFloor is a deployment with no published host floor: no box can arm |
| 106 | // its guard, so none is enrolled automatically. |
| 107 | var ErrNoFloor = errors.New("hostactors: no host floor is published, so no box is enrolled automatically") |
| 108 | |
| 109 | // EnrollPass takes one step for every box whose identity record is |
| 110 | // published and that the roster does not list. It returns the steps in |
| 111 | // reference order, and the first error that stopped the pass as a whole. |
| 112 | func (e *AutoEnroller) EnrollPass(ctx context.Context) ([]EnrollStep, error) { |
| 113 | o, floor, err := e.inputs(ctx) |
| 114 | if err != nil { |
| 115 | return nil, err |
| 116 | } |
| 117 | var steps []EnrollStep |
| 118 | keys, err := e.Directory.Store.List(ctx, boxid.Prefix) |
| 119 | if err != nil { |
| 120 | return nil, err |
| 121 | } |
| 122 | var refs []string |
| 123 | for _, k := range keys { |
| 124 | ref, ok := strings.CutSuffix(strings.TrimPrefix(k, boxid.Prefix), ".json") |
| 125 | if !ok || strings.Contains(ref, "/") { |
| 126 | continue |
| 127 | } |
| 128 | if _, listed := o.Roster.Boxes[ref]; !listed { |
| 129 | refs = append(refs, ref) |
| 130 | } |
| 131 | } |
| 132 | sort.Strings(refs) |
| 133 | if e.attempts == nil { |
| 134 | e.attempts = map[string]attemptSeen{} |
| 135 | } |
| 136 | for _, ref := range refs { |
| 137 | next, step := e.step(ctx, o, floor, ref) |
| 138 | if next.ETag != "" { |
| 139 | o = next |
| 140 | } |
| 141 | steps = append(steps, EnrollStep{Box: ref, Step: step}) |
| 142 | } |
| 143 | for ref := range e.attempts { |
| 144 | if !slices.Contains(refs, ref) { |
| 145 | delete(e.attempts, ref) |
| 146 | } |
| 147 | } |
| 148 | return steps, nil |
| 149 | } |
| 150 | |
| 151 | // inputs reads the roster and the host floor it was published under. |
| 152 | func (e *AutoEnroller) inputs(ctx context.Context) (Observed, hostfloor.Verified, error) { |
| 153 | o, err := e.Directory.Read(ctx) |
| 154 | if err != nil { |
| 155 | return Observed{}, hostfloor.Verified{}, err |
| 156 | } |
| 157 | if o.Roster.Publication != "published" { |
| 158 | return Observed{}, hostfloor.Verified{}, ErrNoFloor |
| 159 | } |
| 160 | floor, err := hostfloor.Read(ctx, e.Directory.Store, e.Directory.PublicKey, e.Directory.Bucket) |
| 161 | if err != nil { |
| 162 | return Observed{}, hostfloor.Verified{}, err |
| 163 | } |
| 164 | if floor.Digest != o.Roster.FloorSHA256 { |
| 165 | return Observed{}, hostfloor.Verified{}, ErrRoster |
| 166 | } |
| 167 | return o, floor, nil |
| 168 | } |
| 169 | |
| 170 | // step takes one box's next step. It returns the roster it wrote, if it |
| 171 | // wrote one, and what it did or waits for. |
| 172 | func (e *AutoEnroller) step(ctx context.Context, o Observed, floor hostfloor.Verified, ref string) (Observed, string) { |
| 173 | d, now := e.Directory, e.Now() |
| 174 | raw, err := boxid.Load(ctx, d.Store, ref) |
| 175 | if err != nil { |
| 176 | return Observed{}, fmt.Sprintf("its identity record could not be read: %v", err) |
| 177 | } |
| 178 | name, region, err := groupRegion(o.Roster, raw) |
| 179 | if err != nil { |
| 180 | return Observed{}, err.Error() |
| 181 | } |
| 182 | if !slices.Contains(region.Namespaces, e.Namespace) || !slices.Contains(region.Pools, e.Pool) { |
| 183 | return Observed{}, fmt.Sprintf("%s does not serve namespace %s and pool %s", name, e.Namespace, e.Pool) |
| 184 | } |
| 185 | v := boxid.Verifier{Account: region.Account, Certs: e.Certs} |
| 186 | expect := boxid.Expect{Box: ref, AWSRegion: region.AWSRegion} |
| 187 | for other, b := range o.Roster.Boxes { |
| 188 | expect.Roster = append(expect.Roster, boxid.Enrolled{Box: other, Cloud: b.Cloud}) |
| 189 | } |
| 190 | ev, err := v.Enroll(raw, expect) |
| 191 | if err != nil { |
| 192 | return Observed{}, fmt.Sprintf("its identity record does not verify: %v", err) |
| 193 | } |
| 194 | instance := ev.Cloud.InstanceID |
| 195 | // A group box publishes under the reference its instance derives; |
| 196 | // the pass derives it again and never takes the key the box chose |
| 197 | // (census ruling region-ha-2b). |
| 198 | if ref != GroupBoxReference(region.Account, instance) { |
| 199 | return Observed{}, "its record is not under its group reference, so it is not enrolled automatically" |
| 200 | } |
| 201 | in := EnrollmentRequest{Reference: ref, Namespace: e.Namespace, Label: ref, Region: name, Pool: e.Pool, AWSRegion: region.AWSRegion} |
| 202 | |
| 203 | // The invitation is only read here, once, and verified for this |
| 204 | // record (enrollmentNonce writes nothing). Invite is get-or-create, so |
| 205 | // it runs only on the bound store after a membership read |
| 206 | // (confirmAndWrite): an invitation that vanished between two reads is |
| 207 | // never minted again without one. |
| 208 | nonce, err := d.enrollmentNonce(ctx, in, raw) |
| 209 | invited := err == nil |
| 210 | switch { |
| 211 | case err == nil, errors.Is(err, os.ErrNotExist): |
| 212 | case errors.Is(err, ErrRoster): |
| 213 | return Observed{}, fmt.Sprintf("its invitation is not for this record: %v", err) |
| 214 | default: |
| 215 | return Observed{}, fmt.Sprintf("its invitation could not be read: %v", err) |
| 216 | } |
| 217 | if invited { |
| 218 | // The report is read under the reference derived from the |
| 219 | // attempt's proven instance (PrivateBoxReport, Group set). |
| 220 | rep, err := e.Evidence.BoxReport(ctx, ref, Box{Namespace: e.Namespace, Label: ref, Group: region.GroupName, Cloud: ev.Cloud}) |
| 221 | if err != nil { |
| 222 | return Observed{}, "waiting for its guard-ready report" |
| 223 | } |
| 224 | if err := VerifyReport(ctx, e.Evidence, d.PublicKey, floor, rep, nonce, instance, now); err != nil { |
| 225 | return Observed{}, "waiting for its guard-ready report" |
| 226 | } |
| 227 | } |
| 228 | a, ok := o.Roster.EnrollmentAttempts[ref] |
| 229 | usable := ok && !a.Consumed && a.InstanceID == instance && a.EvidenceSHA256 == hostfloor.Digest(raw) && a.Region == name && |
| 230 | a.Account == region.Account && a.AWSRegion == region.AWSRegion && a.GroupName == region.GroupName && a.GroupARN == region.GroupARN |
| 231 | seen := e.attempts[ref] |
| 232 | if usable { |
| 233 | if seen.id != a.AttemptID { |
| 234 | seen = attemptSeen{id: a.AttemptID, since: now, tries: seen.tries} |
| 235 | e.attempts[ref] = seen |
| 236 | } |
| 237 | if seen.refused != "" { |
| 238 | // The cloud refused this attempt: keep it until its wait is |
| 239 | // over, asking nothing more, then begin another. |
| 240 | if now.Sub(seen.since) < e.wait(seen.tries) { |
| 241 | return Observed{}, "not confirmed (" + seen.refused + "); waiting before another attempt" |
| 242 | } |
| 243 | } else { |
| 244 | in.AttemptID, in.ConfirmationID = a.AttemptID, a.ConfirmationID |
| 245 | next, step, err := e.confirmAndWrite(ctx, o, v, in, raw, instance, invited, name, region) |
| 246 | if err == nil { |
| 247 | if !invited { |
| 248 | seen.tries = 0 |
| 249 | e.attempts[ref] = seen |
| 250 | } else { |
| 251 | delete(e.attempts, ref) |
| 252 | } |
| 253 | return next, step |
| 254 | } |
| 255 | if !errors.Is(err, ErrNotMember) && !errors.Is(err, ErrMembershipStale) { |
| 256 | // Unknown: a failed read, or a write that failed. The same |
| 257 | // attempt is asked again next time; the wait does not grow. |
| 258 | return Observed{}, step |
| 259 | } |
| 260 | seen.refused, seen.since = err.Error(), now |
| 261 | e.attempts[ref] = seen |
| 262 | return Observed{}, "not confirmed (" + seen.refused + "); waiting before another attempt" |
| 263 | } |
| 264 | } |
| 265 | next, a, err := d.BeginEnrollmentAttempt(ctx, e.Signer, o, in, raw, instance) |
| 266 | if err != nil { |
| 267 | return Observed{}, fmt.Sprintf("an attempt could not be begun: %v", err) |
| 268 | } |
| 269 | e.attempts[ref] = attemptSeen{id: a.AttemptID, since: now, tries: seen.tries + 1} |
| 270 | return next, "began an attempt" |
| 271 | } |
| 272 | |
| 273 | // confirmAndWrite reads the box's membership and makes the write it |
| 274 | // allows: the invitation, or, for a box already invited and guard-ready, |
| 275 | // the enrollment. The membership read is the console's own, made here, |
| 276 | // immediately before the write. |
| 277 | func (e *AutoEnroller) confirmAndWrite(ctx context.Context, o Observed, v boxid.Verifier, in EnrollmentRequest, raw []byte, instance string, invited bool, name string, region Region) (Observed, string, error) { |
| 278 | d := e.Directory |
| 279 | if invited { |
| 280 | next, ev, err := d.EnrollAutomatic(ctx, e.Signer, o, v, in, raw, instance, e.Members, e.Now) |
| 281 | // A write the deadline left unknown (ErrWriteDeadline) is settled |
| 282 | // by the next pass's roster read: a box the roster lists is |
| 283 | // enrolled, and the pass has nothing more to do for it; one it |
| 284 | // does not list goes on with the same attempt, whose wait has not |
| 285 | // grown. |
| 286 | e.logRead(in, instance, ev, "enrolled", err) |
| 287 | if err != nil { |
| 288 | return Observed{}, fmt.Sprintf("it could not be enrolled: %v", err), err |
| 289 | } |
| 290 | return next, fmt.Sprintf("enrolled in %s with the group's role, %s", name, RoleName(region.Role)), nil |
| 291 | } |
| 292 | // checkConfirmed returns the confirmation its read built, and the |
| 293 | // write is bound to that, never to one rebuilt from exported evidence. |
| 294 | _, c, err := d.checkConfirmed(ctx, o, v, in, raw, instance, e.Members, e.Now) |
| 295 | ev := c.evidence() |
| 296 | if err != nil { |
| 297 | e.logRead(in, instance, ev, "", err) |
| 298 | return Observed{}, fmt.Sprintf("its membership could not be confirmed: %v", err), err |
| 299 | } |
| 300 | // Invite is main's and sealed. It runs on the store bound to the |
| 301 | // read's deadline (writeBound), the single enforcement point: its |
| 302 | // record read, its signature's Put and any read-back are refused once |
| 303 | // the read is a minute old; a call in flight then, or still unanswered |
| 304 | // 15 seconds after it began, is cut off as unknown, and so is one that |
| 305 | // answers only after the deadline. The next pass settles it from the |
| 306 | // bucket. |
| 307 | // Nothing that waits sits between the read and this call. |
| 308 | bd, bctx, cancel := writeBound(ctx, d, c, e.Now) |
| 309 | defer cancel() |
| 310 | _, err = bd.Invite(bctx, e.Signer, in.Reference, raw) |
| 311 | e.logRead(in, instance, ev, "invited", err) |
| 312 | if err != nil { |
| 313 | return Observed{}, fmt.Sprintf("it could not be invited: %v", err), err |
| 314 | } |
| 315 | return Observed{}, "invited", nil |
| 316 | } |
| 317 | |
| 318 | // logRead records one membership read: the box, its attempt, its instance, |
| 319 | // the cloud's two request ids and the read's time, and what came of it |
| 320 | // (C2's audit: the roster has no field for them): done on success, |
| 321 | // "refused: " and the reason for the cloud's positive answer that counts |
| 322 | // against the attempt (ErrNotMember, ErrMembershipStale), and "unknown: " |
| 323 | // and the reason for everything else. A call that failed before the cloud |
| 324 | // answered has no request ids and says so. A check that failed before |
| 325 | // any read was made (no read time) is not a membership read and is not |
| 326 | // logged here. |
| 327 | func (e *AutoEnroller) logRead(in EnrollmentRequest, instance string, ev MembershipEvidence, done string, err error) { |
| 328 | if ev.ObservedAt.IsZero() { |
| 329 | return |
| 330 | } |
| 331 | outcome := done |
| 332 | if err != nil { |
| 333 | outcome = "unknown: " + err.Error() |
| 334 | if errors.Is(err, ErrNotMember) || errors.Is(err, ErrMembershipStale) { |
| 335 | outcome = "refused: " + err.Error() |
| 336 | } |
| 337 | } |
| 338 | e.logOutcome(in, instance, ev, outcome) |
| 339 | } |
| 340 | |
| 341 | // logOutcome is one membership log line (S5): the box, its attempt, its |
| 342 | // instance, both request ids and the read's time, and the outcome. |
| 343 | func (e *AutoEnroller) logOutcome(in EnrollmentRequest, instance string, ev MembershipEvidence, outcome string) { |
| 344 | if e.Logf == nil { |
| 345 | return |
| 346 | } |
| 347 | ids := "none" |
| 348 | if len(ev.RequestIDs) > 0 { |
| 349 | ids = strings.Join(ev.RequestIDs, ", ") |
| 350 | } |
| 351 | e.Logf("console: automatic box enrollment: membership read for %s (attempt %s, instance %s, request ids %s, read at %s): %s", in.Reference, in.AttemptID, instance, ids, ev.ObservedAt.Format(time.RFC3339), outcome) |
| 352 | } |
| 353 | |
| 354 | // groupRegion is the one roster region whose group could have launched |
| 355 | // the box the record names: its account and AWS region, with a group. |
| 356 | // The record is read here only to choose; Verifier.Enroll then proves it. |
| 357 | func groupRegion(r Roster, raw []byte) (string, Region, error) { |
| 358 | var rec boxid.Record |
| 359 | if json.Unmarshal(raw, &rec) != nil { |
| 360 | return "", Region{}, errors.New("its identity record is not JSON") |
| 361 | } |
| 362 | doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument) |
| 363 | if err != nil { |
| 364 | return "", Region{}, errors.New("its identity document is not base64") |
| 365 | } |
| 366 | var id struct { |
| 367 | AccountID string `json:"accountId"` |
| 368 | Region string `json:"region"` |
| 369 | } |
| 370 | if json.Unmarshal(doc, &id) != nil { |
| 371 | return "", Region{}, errors.New("its identity document is not JSON") |
| 372 | } |
| 373 | var names []string |
| 374 | for name, g := range r.Regions { |
| 375 | if g.GroupName != "" && g.Account == id.AccountID && g.AWSRegion == id.Region { |
| 376 | names = append(names, name) |
| 377 | } |
| 378 | } |
| 379 | switch len(names) { |
| 380 | case 0: |
| 381 | return "", Region{}, fmt.Errorf("no region's group runs in %s for account %s", id.Region, id.AccountID) |
| 382 | case 1: |
| 383 | return names[0], r.Regions[names[0]], nil |
| 384 | } |
| 385 | sort.Strings(names) |
| 386 | return "", Region{}, fmt.Errorf("the regions %s all have a group in %s", strings.Join(names, ", "), id.Region) |
| 387 | } |
| 388 | |
amadan