| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/rand" |
| 6 | "crypto/sha256" |
| 7 | "encoding/hex" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "github.com/carlosframework/platform/internal/hostactors" |
| 11 | "github.com/carlosframework/platform/internal/hostfloor" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | "log" |
| 14 | "net/http" |
| 15 | "os" |
| 16 | "strings" |
| 17 | "time" |
| 18 | ) |
| 19 | |
| 20 | // activationHeldError names the executable activation reservation that |
| 21 | // blocks a channel (I-4), so the operator learns what to wait for or |
| 22 | // cancel. It is still hostactors.ErrRoster to every errors.Is check. |
| 23 | type activationHeldError struct{ id, component, channel string } |
| 24 | |
| 25 | func (e activationHeldError) Error() string { |
| 26 | return "activation " + e.id + " is held for " + e.component + "/" + e.channel + ". It is released when every reserved box reports completion or is excluded by a termination fence or quarantine. If it was never published, cancel it with carlos system cancel-activation" |
| 27 | } |
| 28 | |
| 29 | func (e activationHeldError) Unwrap() error { return hostactors.ErrRoster } |
| 30 | |
| 31 | func (s *Server) reserveHostUpdate(ctx context.Context, rel *release.SystemRelease, channel, mode string, options release.SystemPromoteOptions, expectedETag ...string) (string, error) { |
| 32 | pub := release.RootSigner(s.cfg.Key).RootPublicKey() |
| 33 | floor, err := hostfloor.Read(ctx, s.cfg.Releases, pub, s.cfg.Bucket) |
| 34 | if errors.Is(err, os.ErrNotExist) { |
| 35 | // Legacy mode only when BOTH the floor and its authorization are |
| 36 | // absent; any incomplete pair is an unfinished publication, never |
| 37 | // the old deployment mode (review round 4). |
| 38 | for _, key := range []string{hostfloor.Key, hostfloor.AuthKey} { |
| 39 | if _, _, e := s.cfg.Releases.Get(ctx, key); !errors.Is(e, os.ErrNotExist) { |
| 40 | return "", hostfloor.ErrFloor |
| 41 | } |
| 42 | } |
| 43 | return "", nil |
| 44 | } |
| 45 | if err != nil { |
| 46 | return "", err |
| 47 | } |
| 48 | tag := strings.TrimPrefix(rel.Source, "release:") |
| 49 | if rel.ReleaseTag != "" { |
| 50 | tag = rel.ReleaseTag |
| 51 | } |
| 52 | if rel.Source == "file" { |
| 53 | keys, e := s.cfg.Releases.List(ctx, "control/system/router/releases/") |
| 54 | if e != nil { |
| 55 | return "", e |
| 56 | } |
| 57 | tag = "" |
| 58 | for _, key := range keys { |
| 59 | raw, _, e := s.cfg.Releases.Get(ctx, key) |
| 60 | if e != nil { |
| 61 | return "", e |
| 62 | } |
| 63 | identity, e := release.VerifySystemIdentity(pub, raw) |
| 64 | if e != nil { |
| 65 | return "", e |
| 66 | } |
| 67 | if identity.Current.Version != rel.Version { |
| 68 | continue |
| 69 | } |
| 70 | matches := true |
| 71 | for platform, artifact := range rel.Artifacts { |
| 72 | if identity.Current.Artifacts[platform] != artifact { |
| 73 | matches = false |
| 74 | } |
| 75 | } |
| 76 | if matches { |
| 77 | if tag != "" && tag != identity.Current.ReleaseTag { |
| 78 | return "", hostfloor.ErrFloor |
| 79 | } |
| 80 | tag = identity.Current.ReleaseTag |
| 81 | } |
| 82 | } |
| 83 | } |
| 84 | approved, err := release.GetSystemIdentity(ctx, s.cfg.Releases, pub, tag) |
| 85 | if err != nil { |
| 86 | return "", err |
| 87 | } |
| 88 | if approved.Current.Version != rel.Version { |
| 89 | return "", hostfloor.ErrFloor |
| 90 | } |
| 91 | for platform, artifact := range rel.Artifacts { |
| 92 | if approved.Current.Artifacts[platform] != artifact { |
| 93 | return "", hostfloor.ErrFloor |
| 94 | } |
| 95 | } |
| 96 | rel.ReleaseTag = approved.Current.ReleaseTag |
| 97 | rel.MovesProtocol = approved.Current.MovesProtocol |
| 98 | rel.ConsoleArtifacts = approved.Current.ConsoleArtifacts |
| 99 | d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: pub, Bucket: s.cfg.Bucket} |
| 100 | o, err := d.Read(ctx) |
| 101 | if err != nil { |
| 102 | return "", err |
| 103 | } |
| 104 | // M-8: a floor binds host updates only through a roster published at |
| 105 | // exactly its digest. A preparing, unpublished or differently bound |
| 106 | // roster is an unfinished publication, never permission to reserve. |
| 107 | if o.Roster.Publication != "published" || o.Roster.FloorSHA256 != floor.Digest { |
| 108 | return "", hostfloor.ErrFloor |
| 109 | } |
| 110 | requestRelease := *rel |
| 111 | requestRelease.ReleasedAt = time.Time{} |
| 112 | request, err := json.Marshal(struct { |
| 113 | Channel string |
| 114 | Mode string |
| 115 | Release release.SystemRelease |
| 116 | Spread time.Duration |
| 117 | }{channel, mode, requestRelease, options.Spread}) |
| 118 | if err != nil { |
| 119 | return "", err |
| 120 | } |
| 121 | digest := sha256.Sum256(request) |
| 122 | requestID := hex.EncodeToString(digest[:]) |
| 123 | for id, existing := range o.Roster.Activations { |
| 124 | if existing.Publication == nil || existing.Publication.Channel != channel { |
| 125 | continue |
| 126 | } |
| 127 | if existing.RequestSHA256 != requestID || existing.FloorSHA256 != floor.Digest { |
| 128 | return "", activationHeldError{id: id, component: existing.Publication.Component, channel: channel} |
| 129 | } |
| 130 | if _, err := existing.Publication.Verify(pub); err != nil { |
| 131 | return "", err |
| 132 | } |
| 133 | return id, nil |
| 134 | } |
| 135 | var nonce [16]byte |
| 136 | if _, err = rand.Read(nonce[:]); err != nil { |
| 137 | return "", err |
| 138 | } |
| 139 | id := hex.EncodeToString(nonce[:]) |
| 140 | var publication release.SystemPublication |
| 141 | if mode == "rollback" { |
| 142 | publication, err = release.PrepareSystemRollback(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), "router", channel, options.Actor, id, expectedETag...) |
| 143 | } else { |
| 144 | publication, err = release.PrepareSystemPromotion(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), "router", channel, *rel, options, id) |
| 145 | } |
| 146 | if err != nil { |
| 147 | return "", err |
| 148 | } |
| 149 | h := hostactors.Reservation{Publication: &publication, RequestSHA256: requestID, UpdateMode: mode, ID: id, FloorSHA256: floor.Digest, Actors: map[string]string{}, Releases: map[string]hostactors.Report{}, Hashes: map[string][]string{}} |
| 150 | e := hostactors.ExactEvidence{PublicKey: d.PublicKey, Store: s.cfg.Releases, Boxes: d.PrivateBoxReport} |
| 151 | for ref, b := range o.Roster.Boxes { |
| 152 | if b.State != "active" { |
| 153 | continue |
| 154 | } |
| 155 | report, err := d.PrivateBoxReport(ctx, ref, b) |
| 156 | if err != nil { |
| 157 | return "", err |
| 158 | } |
| 159 | if err = hostactors.VerifyReport(ctx, e, pub, floor, report, b.Enrollment, b.Cloud.InstanceID, time.Now()); err != nil { |
| 160 | return "", err |
| 161 | } |
| 162 | if report.SystemChannel == "" { |
| 163 | return "", hostfloor.ErrFloor |
| 164 | } |
| 165 | if report.SystemChannel != channel { |
| 166 | continue |
| 167 | } |
| 168 | target, ok := rel.Artifacts[report.Platform] |
| 169 | if !ok { |
| 170 | return "", hostfloor.ErrFloor |
| 171 | } |
| 172 | identityKey, _ := release.IdentityKey(rel.ReleaseTag) |
| 173 | identity, _, err := s.cfg.Releases.Get(ctx, identityKey) |
| 174 | if err != nil { |
| 175 | return "", err |
| 176 | } |
| 177 | if err = hostfloor.CheckImage(pub, floor, identity, rel.Version, report.Platform, target.SHA256); err != nil { |
| 178 | return "", err |
| 179 | } |
| 180 | if target.SHA256 == report.BinarySHA256 { |
| 181 | continue |
| 182 | } |
| 183 | key := "box:" + ref |
| 184 | h.Actors[key] = b.Enrollment |
| 185 | h.Releases[key] = report |
| 186 | // This transaction refreshes recovery from the authenticated current |
| 187 | // image. A box unable to do that retains the reservation and refuses. |
| 188 | h.Hashes[key] = []string{report.BinarySHA256, target.SHA256, report.BinarySHA256} |
| 189 | } |
| 190 | if len(h.Actors) == 0 { |
| 191 | return "", nil |
| 192 | } |
| 193 | _, err = d.Reserve(ctx, release.RootSigner(s.cfg.Key), o, h, true) |
| 194 | return id, err |
| 195 | } |
| 196 | |
| 197 | func (s *Server) promoteHostUpdate(ctx context.Context, rel release.SystemRelease, component, channel string, options release.SystemPromoteOptions) (*release.SystemPointer, error) { |
| 198 | id, err := s.reserveHostUpdate(ctx, &rel, channel, "promote", options) |
| 199 | if err != nil { |
| 200 | return nil, err |
| 201 | } |
| 202 | if id == "" { |
| 203 | return release.PromoteSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), component, channel, rel, options) |
| 204 | } |
| 205 | d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: release.RootSigner(s.cfg.Key).RootPublicKey(), Bucket: s.cfg.Bucket} |
| 206 | return d.PublishActivation(ctx, id) |
| 207 | } |
| 208 | |
| 209 | func (s *Server) rollbackHostUpdate(ctx context.Context, component, channel, actor string) (*release.SystemPointer, error) { |
| 210 | d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: release.RootSigner(s.cfg.Key).RootPublicKey(), Bucket: s.cfg.Bucket} |
| 211 | floor, err := hostfloor.Read(ctx, s.cfg.Releases, d.PublicKey, s.cfg.Bucket) |
| 212 | if err == nil { |
| 213 | roster, err := d.Read(ctx) |
| 214 | if err != nil { |
| 215 | return nil, err |
| 216 | } |
| 217 | // A successful publication with a lost response must not pop history again. |
| 218 | for id, h := range roster.Roster.Activations { |
| 219 | if h.Publication == nil || h.Publication.Channel != channel { |
| 220 | continue |
| 221 | } |
| 222 | if h.UpdateMode != "rollback" || h.FloorSHA256 != floor.Digest { |
| 223 | return nil, activationHeldError{id: id, component: component, channel: channel} |
| 224 | } |
| 225 | return d.PublishActivation(ctx, id) |
| 226 | } |
| 227 | } else if !errors.Is(err, os.ErrNotExist) { |
| 228 | return nil, err |
| 229 | } |
| 230 | current, etag, err := release.GetSystemPointer(ctx, s.cfg.Releases, d.PublicKey, component, channel) |
| 231 | if err != nil { |
| 232 | return nil, err |
| 233 | } |
| 234 | if len(current.History) == 0 { |
| 235 | return nil, release.ErrNoSystemHistory |
| 236 | } |
| 237 | target := current.History[0] |
| 238 | id, err := s.reserveHostUpdate(ctx, &target, channel, "rollback", release.SystemPromoteOptions{Actor: actor}, etag) |
| 239 | if err != nil { |
| 240 | return nil, err |
| 241 | } |
| 242 | if id == "" { |
| 243 | return release.RollbackSystem(ctx, s.cfg.Releases, release.RootSigner(s.cfg.Key), component, channel, actor, etag) |
| 244 | } |
| 245 | return d.PublishActivation(ctx, id) |
| 246 | } |
| 247 | |
| 248 | // handleAPISystemActivationCancel is the operator's way out of an executable |
| 249 | // activation reservation that was never published (I-4). Such a reservation |
| 250 | // blocks rollback and every other promote of its channel, and nothing else |
| 251 | // releases it: no box ever saw it, so no box can report completing it. |
| 252 | // |
| 253 | // It takes the update doors' gate (apiSystemRoller: a person who owns the |
| 254 | // ops account or is on CARLOS_SYSTEM_OPERATORS) and the signing key. The |
| 255 | // channel names the reservation, since at most one is held per channel; an |
| 256 | // activation ID, when given, must be the one held. CancelActivation fences |
| 257 | // the pointer at the publication's expected ETag before it removes the |
| 258 | // reservation, so a delayed publication can never land afterwards, and a |
| 259 | // publication that already landed answers 409: a published activation is |
| 260 | // released only by completion or the D7 exclusions, never by this door. |
| 261 | func (s *Server) handleAPISystemActivationCancel(w http.ResponseWriter, r *http.Request) { |
| 262 | addr, _, ok := s.apiSystemRoller(w, r) |
| 263 | if !ok { |
| 264 | return |
| 265 | } |
| 266 | if !s.systemBucket(w) { |
| 267 | return |
| 268 | } |
| 269 | if s.cfg.Key == nil { |
| 270 | jsonError(w, http.StatusServiceUnavailable, "no signing key on this host, so the console is running read-only") |
| 271 | return |
| 272 | } |
| 273 | var in struct { |
| 274 | Component string `json:"component"` |
| 275 | Channel string `json:"channel"` |
| 276 | Activation string `json:"activation"` |
| 277 | } |
| 278 | if !readJSONStrictLimit(w, r, &in, 1<<10) { |
| 279 | return |
| 280 | } |
| 281 | if in.Component == "" { |
| 282 | in.Component = release.SystemComponentRouter |
| 283 | } |
| 284 | if in.Component != release.SystemComponentRouter { |
| 285 | jsonError(w, http.StatusBadRequest, "unknown component "+in.Component) |
| 286 | return |
| 287 | } |
| 288 | if in.Channel != release.RungCanary && in.Channel != release.RungStable { |
| 289 | jsonError(w, http.StatusBadRequest, "channel must be canary or stable") |
| 290 | return |
| 291 | } |
| 292 | target := in.Component + "/" + in.Channel |
| 293 | s.systemMu.Lock() |
| 294 | running := s.systemRunning[target] |
| 295 | s.systemMu.Unlock() |
| 296 | if running != "" { |
| 297 | writeJSON(w, http.StatusConflict, map[string]string{ |
| 298 | "error": "a promote of " + target + " is running on this console; watch it or wait for it to finish", |
| 299 | "job": running, |
| 300 | }) |
| 301 | return |
| 302 | } |
| 303 | ctx := r.Context() |
| 304 | sg := release.RootSigner(s.cfg.Key) |
| 305 | d := hostactors.Directory{Store: s.cfg.Releases, PublicKey: sg.RootPublicKey(), Bucket: s.cfg.Bucket} |
| 306 | o, err := d.Read(ctx) |
| 307 | if errors.Is(err, os.ErrNotExist) { |
| 308 | jsonError(w, http.StatusNotFound, "no activation reservation is held for "+target) |
| 309 | return |
| 310 | } |
| 311 | if err != nil { |
| 312 | log.Printf("console: cancel activation %s: read roster: %v", target, err) |
| 313 | jsonError(w, http.StatusInternalServerError, "could not read the actor roster; see the console log") |
| 314 | return |
| 315 | } |
| 316 | id := "" |
| 317 | for key, h := range o.Roster.Activations { |
| 318 | if h.Publication != nil && h.Publication.Component == in.Component && h.Publication.Channel == in.Channel { |
| 319 | id = key |
| 320 | } |
| 321 | } |
| 322 | if id == "" { |
| 323 | jsonError(w, http.StatusNotFound, "no activation reservation is held for "+target) |
| 324 | return |
| 325 | } |
| 326 | if in.Activation != "" && in.Activation != id { |
| 327 | jsonError(w, http.StatusConflict, "the reservation held for "+target+" is "+id+", not "+in.Activation+"; nothing was cancelled") |
| 328 | return |
| 329 | } |
| 330 | err = d.CancelActivation(ctx, sg, id) |
| 331 | switch { |
| 332 | case errors.Is(err, release.ErrCASConflict): |
| 333 | jsonError(w, http.StatusConflict, "activation "+id+" was already published, or its channel pointer has moved, so it cannot be cancelled. It is released when every reserved box reports completion or is excluded by a termination fence or quarantine") |
| 334 | return |
| 335 | case errors.Is(err, hostactors.ErrRoster): |
| 336 | jsonError(w, http.StatusConflict, "activation "+id+" changed while it was being cancelled; nothing was released. Check carlos system status and try again") |
| 337 | return |
| 338 | case err != nil: |
| 339 | log.Printf("console: cancel activation %s on %s: %v", id, target, err) |
| 340 | jsonError(w, http.StatusInternalServerError, "could not cancel activation "+id+"; see the console log") |
| 341 | return |
| 342 | } |
| 343 | s.auditSystem(addr, "system-activation-cancel", "cancelled activation "+id+" on "+target) |
| 344 | writeJSON(w, http.StatusOK, map[string]string{"component": in.Component, "channel": in.Channel, "activation": id, "state": "cancelled"}) |
| 345 | } |
| 346 | |
amadan