| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/base64" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "net/http" |
| 10 | "os" |
| 11 | "runtime" |
| 12 | "strings" |
| 13 | "sync/atomic" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/boxid" |
| 17 | "github.com/carlosframework/platform/internal/hostactors" |
| 18 | "github.com/carlosframework/platform/internal/hostfloor" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | "github.com/carlosframework/platform/internal/status" |
| 21 | "github.com/carlosframework/platform/internal/version" |
| 22 | ) |
| 23 | |
| 24 | // Joining and hosting (region HA spec 2 §5, A1-2, A2-4). |
| 25 | // |
| 26 | // With CARLOS_HOST_CLAIM and CARLOS_REGION_HA both on, the agent chooses |
| 27 | // its mode at every start, from three facts and never from one file's |
| 28 | // presence. It hosts only when all three hold: |
| 29 | // |
| 30 | // - /etc/carlos/host-actor.json names this box; |
| 31 | // - the signed roster has an active entry for that enrollment and |
| 32 | // instance; |
| 33 | // - the host lifecycle guard is armed under the roster's floor, with |
| 34 | // this process's own binary one it admits. |
| 35 | // |
| 36 | // Anything else is joining. A joining box hosts nothing, publishes no |
| 37 | // edges file, and answers the load balancer 503 (joining.go). Joining |
| 38 | // resumes from whatever it finds, and each stage names the next step: |
| 39 | // |
| 40 | // - no invitation: the console has not invited the box yet; |
| 41 | // - invited: root arms the guard (carlos system join, joinroot.go); |
| 42 | // - guard installed: the box publishes its guard-ready report; |
| 43 | // - guard-ready reported: the console enrolls the box; |
| 44 | // - held: the roster names this box, but not as an active entry for its |
| 45 | // enrollment and armed guard, or, for a box its region's group |
| 46 | // launched, the instance was stopped and started since the identity |
| 47 | // record it was enrolled on (C3: it must prove its group membership |
| 48 | // again, and never hosts on its old enrollment). Nothing automatic |
| 49 | // can fix that: a held group box answers 503 and the group replaces |
| 50 | // it; an operator sorts out any other. |
| 51 | // |
| 52 | // A fact that cannot be read is never taken as its negative: the mode is |
| 53 | // then unknown (errModeUnknown), and the caller asks again later. |
| 54 | |
| 55 | type agentMode string |
| 56 | |
| 57 | const ( |
| 58 | modeHosting agentMode = "hosting" |
| 59 | modeJoining agentMode = "joining" |
| 60 | ) |
| 61 | |
| 62 | type joinStage string |
| 63 | |
| 64 | const ( |
| 65 | stageNoInvitation joinStage = "no invitation" |
| 66 | stageInvited joinStage = "invited" |
| 67 | stageGuardInstalled joinStage = "guard installed" |
| 68 | stageGuardReady joinStage = "guard-ready reported" |
| 69 | stageHeld joinStage = "held" |
| 70 | ) |
| 71 | |
| 72 | // modeDecision is one start's answer. Stage and Why are joining's. |
| 73 | type modeDecision struct { |
| 74 | Mode agentMode |
| 75 | Stage joinStage |
| 76 | Box string |
| 77 | Why string |
| 78 | } |
| 79 | |
| 80 | // errModeUnknown is a fact that could not be read: no mode is chosen. |
| 81 | var errModeUnknown = errors.New("agent mode unknown") |
| 82 | |
| 83 | func modeUnknown(what string, err error) error { |
| 84 | return fmt.Errorf("%w: %s could not be read: %v", errModeUnknown, what, err) |
| 85 | } |
| 86 | |
| 87 | // reportFreshness is how old a guard-ready report may be and still count: |
| 88 | // the console's own limit (hostactors.VerifyReport). |
| 89 | const reportFreshness = 15 * time.Minute |
| 90 | |
| 91 | // hostActorPath is the box's enrollment, written by guard-arm. |
| 92 | const hostActorPath = "/etc/carlos/host-actor.json" |
| 93 | |
| 94 | // modeFacts are where selectMode reads from; production's are |
| 95 | // productionModeFacts. |
| 96 | type modeFacts struct { |
| 97 | // box is this box's reference: the label its identity record and its |
| 98 | // status report are published under. |
| 99 | box string |
| 100 | // enrollment reads host-actor.json; os.ErrNotExist when there is none. |
| 101 | enrollment func() (hostfloor.Enrollment, error) |
| 102 | // roster reads the signed roster. |
| 103 | roster func(context.Context) (hostactors.Roster, error) |
| 104 | // guard is the guard's view of this process (hostfloor.Disk.Running). |
| 105 | // hostfloor.ErrFloor or os.ErrNotExist: not armed, or not for this |
| 106 | // binary. |
| 107 | guard func() (hostfloor.Running, error) |
| 108 | // invited reports whether the console has written box's invitation. |
| 109 | invited func(context.Context, string) (bool, error) |
| 110 | // report reads box's own published status report; os.ErrNotExist when |
| 111 | // there is none. |
| 112 | report func(context.Context, string) (status.Box, error) |
| 113 | // relaunch says why this instance is not the launch the roster entry |
| 114 | // row was enrolled on ("" when it is): asked only of a box its |
| 115 | // region's group launched (row.Group set). |
| 116 | relaunch func(context.Context, hostactors.Box) (string, error) |
| 117 | // marked is host.env's group marker (CARLOS_BOX_REFERENCE=group), and |
| 118 | // region its CARLOS_REGION: values from the parameter store, which |
| 119 | // select this box's authority, so they are checked against the signed |
| 120 | // roster before the box hosts (crossCheck). |
| 121 | marked bool |
| 122 | region string |
| 123 | now func() time.Time |
| 124 | } |
| 125 | |
| 126 | // selectMode chooses this start's mode. |
| 127 | func selectMode(ctx context.Context, f modeFacts) (modeDecision, error) { |
| 128 | d := modeDecision{Mode: modeJoining, Box: f.box} |
| 129 | e, err := f.enrollment() |
| 130 | if errors.Is(err, os.ErrNotExist) { |
| 131 | invited, err := f.invited(ctx, f.box) |
| 132 | if err != nil { |
| 133 | return d, modeUnknown("the invitation", err) |
| 134 | } |
| 135 | d.Stage = stageNoInvitation |
| 136 | if invited { |
| 137 | d.Stage = stageInvited |
| 138 | } |
| 139 | return d, nil |
| 140 | } |
| 141 | if err != nil { |
| 142 | return d, modeUnknown(hostActorPath, err) |
| 143 | } |
| 144 | if e.Box != f.box { |
| 145 | d.Stage, d.Why = stageHeld, fmt.Sprintf("%s names %s, not this box (%s)", hostActorPath, e.Box, f.box) |
| 146 | return d, nil |
| 147 | } |
| 148 | r, err := f.roster(ctx) |
| 149 | if err != nil { |
| 150 | return d, modeUnknown("the roster", err) |
| 151 | } |
| 152 | g, err := f.guard() |
| 153 | if err != nil && !errors.Is(err, hostfloor.ErrFloor) && !errors.Is(err, os.ErrNotExist) { |
| 154 | return d, modeUnknown("the host lifecycle guard", err) |
| 155 | } |
| 156 | armed := err == nil && g.GuardReady && g.FloorSHA256 == r.FloorSHA256 |
| 157 | row, listed := r.Boxes[e.Box] |
| 158 | switch { |
| 159 | case listed && row.Enrollment == e.Enrollment && row.Cloud.InstanceID == e.InstanceID && row.State == "active" && armed: |
| 160 | if why := crossCheck(f, row); why != "" { |
| 161 | d.Stage, d.Why = stageHeld, why |
| 162 | return d, nil |
| 163 | } |
| 164 | if row.Group != "" { |
| 165 | why, err := f.relaunch(ctx, row) |
| 166 | if err != nil { |
| 167 | return d, modeUnknown("this instance's launch", err) |
| 168 | } |
| 169 | if why != "" { |
| 170 | d.Stage, d.Why = stageHeld, why |
| 171 | return d, nil |
| 172 | } |
| 173 | } |
| 174 | d.Mode, d.Stage = modeHosting, "" |
| 175 | case listed: |
| 176 | d.Stage = stageHeld |
| 177 | d.Why = fmt.Sprintf("the roster's entry for %s is %s, enrollment %s on %s; this box holds enrollment %s on %s, guard armed %t", |
| 178 | e.Box, row.State, row.Enrollment, row.Cloud.InstanceID, e.Enrollment, e.InstanceID, armed) |
| 179 | case !armed: |
| 180 | // guard-arm writes host-actor.json before it persists the armed |
| 181 | // policy: an arm that was cut off is still the invited stage. |
| 182 | d.Stage = stageInvited |
| 183 | default: |
| 184 | rep, err := f.report(ctx, f.box) |
| 185 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 186 | return d, modeUnknown("this box's status report", err) |
| 187 | } |
| 188 | d.Stage = stageGuardInstalled |
| 189 | if err == nil && guardReadyReport(rep, e, g, f.now()) { |
| 190 | d.Stage = stageGuardReady |
| 191 | } |
| 192 | } |
| 193 | return d, nil |
| 194 | } |
| 195 | |
| 196 | // crossCheck says why host.env's authority-selecting values disagree with |
| 197 | // the roster entry the box would host under ("" when they agree). The |
| 198 | // group marker must match whether the console enrolled the box from a |
| 199 | // group, and a group box's CARLOS_REGION must name the region it was |
| 200 | // enrolled in. It is made here, at the agent's use, and not when |
| 201 | // host-env write assembles the file: at boot the box has no roster entry |
| 202 | // to check against (it is not enrolled yet), and before enrollment |
| 203 | // neither value grants anything, since the console derives the box's |
| 204 | // reference and region from its proven instance itself. A box an operator |
| 205 | // enrolled keeps its own CARLOS_REGION, unchecked, as on main. |
| 206 | func crossCheck(f modeFacts, row hostactors.Box) string { |
| 207 | switch { |
| 208 | case f.marked && row.Group == "": |
| 209 | return "host.env marks this box as one its region's group launched, but the roster enrolled it by hand; it does not host" |
| 210 | case !f.marked && row.Group != "": |
| 211 | return "the roster enrolled this box from group " + row.Group + ", but host.env does not mark it so; it does not host" |
| 212 | case f.marked && f.region == "": |
| 213 | return "host.env names no region, but the roster enrolled this box in " + row.Region + "; it does not host" |
| 214 | case f.marked && f.region != row.Region: |
| 215 | return "host.env names region " + f.region + ", but the roster enrolled this box in " + row.Region + "; it does not host" |
| 216 | } |
| 217 | return "" |
| 218 | } |
| 219 | |
| 220 | // relaunched says why current, this instance's identity record read from |
| 221 | // IMDS now, is not the launch row was enrolled on: the record published |
| 222 | // under the box's reference (enrolled) must be the one the roster entry |
| 223 | // binds (its digest), and the two identity documents must name the same |
| 224 | // pendingTime. EC2 sets pendingTime each time an instance enters the |
| 225 | // pending state: at launch, and again at every start after a stop, but |
| 226 | // not at a reboot. "" when they agree. |
| 227 | func relaunched(row hostactors.Box, enrolled []byte, current boxid.Record) string { |
| 228 | if hostfloor.Digest(enrolled) != row.EvidenceSHA256 { |
| 229 | return "the identity record under this box's reference is not the one it was enrolled on" |
| 230 | } |
| 231 | var rec boxid.Record |
| 232 | if json.Unmarshal(enrolled, &rec) != nil { |
| 233 | return "the identity record it was enrolled on cannot be read" |
| 234 | } |
| 235 | then, now := recordPendingTime(rec), recordPendingTime(current) |
| 236 | switch { |
| 237 | case then == "" || now == "": |
| 238 | return "an identity document names no pendingTime, so this launch cannot be matched to its enrollment" |
| 239 | case then != now: |
| 240 | return fmt.Sprintf("this instance was stopped and started since it was enrolled (launched %s, enrolled on the launch of %s): it must prove its group membership again, so it does not host", now, then) |
| 241 | } |
| 242 | return "" |
| 243 | } |
| 244 | |
| 245 | // recordPendingTime is the pendingTime of a record's identity document, "" |
| 246 | // when it has none. |
| 247 | func recordPendingTime(rec boxid.Record) string { |
| 248 | doc, err := base64.StdEncoding.DecodeString(rec.IdentityDocument) |
| 249 | if err != nil { |
| 250 | return "" |
| 251 | } |
| 252 | var d struct { |
| 253 | PendingTime string `json:"pendingTime"` |
| 254 | } |
| 255 | if json.Unmarshal(doc, &d) != nil { |
| 256 | return "" |
| 257 | } |
| 258 | return d.PendingTime |
| 259 | } |
| 260 | |
| 261 | // guardReadyReport reports whether rep is this box's fresh guard-ready |
| 262 | // report for enrollment e under the guard g. |
| 263 | func guardReadyReport(rep status.Box, e hostfloor.Enrollment, g hostfloor.Running, now time.Time) bool { |
| 264 | m := rep.Metrics |
| 265 | return m != nil && m.Enrollment == e.Enrollment && m.FloorGuardReady && m.FloorSHA256 == g.FloorSHA256 && m.BinarySHA256 == g.Hash && |
| 266 | !rep.GeneratedAt.IsZero() && now.Sub(rep.GeneratedAt) <= reportFreshness |
| 267 | } |
| 268 | |
| 269 | // productionModeFacts reads the deployment store, the status store and |
| 270 | // this box's own files. |
| 271 | func productionModeFacts(ctx context.Context, getenv func(string) string) (modeFacts, error) { |
| 272 | store, err := openDeployStoreEnv(ctx, getenv) |
| 273 | if err != nil { |
| 274 | return modeFacts{}, err |
| 275 | } |
| 276 | statusStore, _, err := openStatusStore(ctx, getenv) |
| 277 | if err != nil { |
| 278 | return modeFacts{}, err |
| 279 | } |
| 280 | pub, err := release.ParsePublicKey(getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY") |
| 281 | if err != nil { |
| 282 | return modeFacts{}, err |
| 283 | } |
| 284 | disk, err := floorDisk(release.OptDir()) |
| 285 | if err != nil { |
| 286 | return modeFacts{}, err |
| 287 | } |
| 288 | bucket := getenv("CARLOS_DEPLOYMENT_BUCKET") |
| 289 | box := metricsBoxLabel(getenv, readMachineID()) |
| 290 | if box == "" { |
| 291 | return modeFacts{}, errors.New("this box's group reference could not be read") |
| 292 | } |
| 293 | imds := &http.Client{Timeout: 2 * time.Second} |
| 294 | return modeFacts{ |
| 295 | box: box, |
| 296 | marked: groupReferenced(getenv), |
| 297 | region: strings.TrimSpace(getenv("CARLOS_REGION")), |
| 298 | enrollment: func() (hostfloor.Enrollment, error) { return hostfloor.ReadEnrollment(hostActorPath) }, |
| 299 | roster: func(ctx context.Context) (hostactors.Roster, error) { |
| 300 | return readGuardRoster(ctx, store, pub, bucket) |
| 301 | }, |
| 302 | guard: func() (hostfloor.Running, error) { |
| 303 | return disk.Running(version.Build, runtime.GOOS+"-"+runtime.GOARCH) |
| 304 | }, |
| 305 | invited: func(ctx context.Context, box string) (bool, error) { |
| 306 | _, _, err := store.Get(ctx, invitationKey(box)) |
| 307 | if errors.Is(err, os.ErrNotExist) { |
| 308 | return false, nil |
| 309 | } |
| 310 | return err == nil, err |
| 311 | }, |
| 312 | report: func(ctx context.Context, box string) (status.Box, error) { |
| 313 | raw, _, err := statusStore.Get(ctx, "status/boxes/"+box+".json") |
| 314 | if err != nil { |
| 315 | return status.Box{}, err |
| 316 | } |
| 317 | var b status.Box |
| 318 | if err := json.Unmarshal(raw, &b); err != nil { |
| 319 | return status.Box{}, err |
| 320 | } |
| 321 | return b, nil |
| 322 | }, |
| 323 | relaunch: func(ctx context.Context, row hostactors.Box) (string, error) { |
| 324 | enrolled, err := boxid.Load(ctx, store, box) |
| 325 | if err != nil { |
| 326 | return "", err |
| 327 | } |
| 328 | current, err := readIdentityRecord(imds, imdsBase, box, time.Now()) |
| 329 | if err != nil { |
| 330 | return "", err |
| 331 | } |
| 332 | return relaunched(row, enrolled, current), nil |
| 333 | }, |
| 334 | now: time.Now, |
| 335 | }, nil |
| 336 | } |
| 337 | |
| 338 | // invitationKey is where the console writes box's invitation |
| 339 | // (hostactors.Directory.Invite). |
| 340 | func invitationKey(box string) string { return "control/host-actor-invitations/" + box + ".json" } |
| 341 | |
| 342 | // agentHosting is this process's mode, decided once by RunAgent before |
| 343 | // the agent starts (hostingGate): true for the claim agent, false while |
| 344 | // joining. The claim wiring reports it to the load balancer's health |
| 345 | // answer (ClaimWiring.startRegional). |
| 346 | var agentHosting atomic.Bool |
| 347 | |
amadan