| 1 | // The host-retry door (zero-downtime spec §8.4, §9.1): `carlos instances |
| 2 | // retry <host>` clears one instance's failure hold, and its next attempt |
| 3 | // runs as a canary. The app-wide retry is `carlos promote --retry`, which |
| 4 | // re-signs the channel pointer. |
| 5 | // |
| 6 | // The request rides the app's host-retry log (release.AppendHostRetry, |
| 7 | // readiness round 2, Task 3), which no older console rewrites and no older |
| 8 | // box reads. Every box can write that log (Astra impl-6 finding 5), so the |
| 9 | // entry carries a grant the root key signs here (release.HostRetryGrant, |
| 10 | // spec §11; Paul's ruling on plan 6 Q4, amadan #272), naming the held build |
| 11 | // and its restriction's epoch and book as the box reported them, and a box |
| 12 | // clears nothing on an entry without one. The door writes only for a box |
| 13 | // whose report says it can take the grant (retryHostBuild): an older box, |
| 14 | // or one whose failure book is not saved yet, is refused with a message |
| 15 | // and nothing is written. A console with no key (read-only) answers 503 |
| 16 | // and writes nothing. ROLE: operate, as `carlos restart` is. |
| 17 | package console |
| 18 | |
| 19 | import ( |
| 20 | "context" |
| 21 | "errors" |
| 22 | "fmt" |
| 23 | "log" |
| 24 | "net/http" |
| 25 | "os" |
| 26 | "time" |
| 27 | |
| 28 | "github.com/carlosframework/platform/internal/provision" |
| 29 | "github.com/carlosframework/platform/internal/release" |
| 30 | ) |
| 31 | |
| 32 | const ( |
| 33 | apiRetryStore = "could not write the retry request. Try again" |
| 34 | apiRetryHostAudit = "requested a retry of %s on %s (%s)" |
| 35 | apiRetryNotHeld = "%s is not held back by a failed build, as far as its box last reported, so there is nothing to retry" |
| 36 | // apiRetryNoKey is a read-only console's answer: a retry is a grant the |
| 37 | // root key signs (spec §11), and there is no key here to sign it. |
| 38 | apiRetryNoKey = "no signing key on this host. this console cannot request a retry" |
| 39 | // apiRetryUnsupported answers a box whose report has no retry_v or no |
| 40 | // held_book: it reads no grant from the host-retry log, so a retry |
| 41 | // written for it would clear nothing. |
| 42 | apiRetryUnsupported = "this instance's box is too old to take a retry. update the box, then try again" |
| 43 | // apiRetryUnsaved answers a held host reported at epoch 0: its box's |
| 44 | // failure book owes a save, and no grant clears anything until it lands. |
| 45 | apiRetryUnsaved = "this instance's box has not saved its failure yet. try again in a minute" |
| 46 | ) |
| 47 | |
| 48 | var ( |
| 49 | errRetryNoRecord = errors.New("retry: no such instance") |
| 50 | errRetryNotHeld = errors.New("retry: not held") |
| 51 | errRetryNoKey = errors.New("retry: no signing key on this console") |
| 52 | // errRetryUnsupported is a held host whose box cannot take a retry: |
| 53 | // its report has no retry_v (a box built before the host-retry log) or |
| 54 | // no held_book (one built before the book's identity). |
| 55 | errRetryUnsupported = errors.New("retry: the box cannot take a retry") |
| 56 | // errRetryUnsaved is a held host whose box reports retry_v but epoch 0: |
| 57 | // its failure book is not saved (readiness round 2, Task 1), and a grant |
| 58 | // signed now would name no epoch the box accepts. |
| 59 | errRetryUnsaved = errors.New("retry: the box has not saved its failure book") |
| 60 | ) |
| 61 | |
| 62 | // retryHostBuild is the build holding host back, and the epoch of that |
| 63 | // restriction and the failure book it counts in (plan review finding 13; |
| 64 | // final review round 1, finding 3), as its box reports them. The record |
| 65 | // must exist in this account and app (the instance family's 404), and the |
| 66 | // report must be this app's own (boxJoin.rollout), read through the same |
| 67 | // loader as the list and the instance page (loadRolloutJoin). A box that |
| 68 | // could not arm its scheduler is not held, except one that declined the |
| 69 | // flag (a decline code, rolloutwords.UnavailableDeclines) and reports the |
| 70 | // host held by the failure book it still honours: that box verifies and |
| 71 | // applies a signed retry too (Task 10, plan review #5). A refusing box, or |
| 72 | // one whose code this console does not know, is never retried here. |
| 73 | // |
| 74 | // Then the box's report is asked whether it can take the grant (readiness |
| 75 | // round 2, Task 3), and a retry it could not take is refused with a |
| 76 | // message rather than written to clear nothing: a report with no retry_v |
| 77 | // is a box built before the host-retry log, which never reads it, and one |
| 78 | // with no held_book is older than the book's identity |
| 79 | // (errRetryUnsupported); a held host at epoch 0 is a box whose failure |
| 80 | // book owes a save, and no grant clears anything until it lands |
| 81 | // (errRetryUnsaved). |
| 82 | func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host string) (heldRestriction, error) { |
| 83 | if _, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host); err != nil { |
| 84 | if errors.Is(err, os.ErrNotExist) { |
| 85 | return heldRestriction{}, errRetryNoRecord |
| 86 | } |
| 87 | return heldRestriction{}, err |
| 88 | } |
| 89 | join, _ := s.loadRolloutJoin(ctx, acct.Sqid, time.Now(), nil) |
| 90 | ro := join.rollout(host, acct.Sqid, app) |
| 91 | if !ro.retryable() { |
| 92 | return heldRestriction{}, errRetryNotHeld |
| 93 | } |
| 94 | if err := ro.retryRefusal(); err != nil { |
| 95 | return heldRestriction{}, err |
| 96 | } |
| 97 | return heldRestriction{build: ro.HeldBy, book: ro.heldBook, epoch: ro.heldEpoch, declined: ro.Unavailable != ""}, nil |
| 98 | } |
| 99 | |
| 100 | // heldRestriction is what a retry grant names: the build holding the host |
| 101 | // back, and that restriction's epoch and book as its box reported them. |
| 102 | // declined marks a box that declined the flag: it runs no canary, so the |
| 103 | // browser's confirmation says so. |
| 104 | type heldRestriction struct { |
| 105 | build, book string |
| 106 | epoch uint64 |
| 107 | declined bool |
| 108 | } |
| 109 | |
| 110 | // requestHostRetry signs the retry grant for held (its build, book and |
| 111 | // epoch) with the root key, appends it to the app's host-retry log |
| 112 | // (release.AppendHostRetry), and announces it on the serial. With no key |
| 113 | // it writes nothing (errRetryNoKey): an unsigned entry is one every box |
| 114 | // drops. |
| 115 | func (s *Server) requestHostRetry(ctx context.Context, acct, app, host string, held heldRestriction) (release.RestartRequest, error) { |
| 116 | if s.cfg.Key == nil { |
| 117 | return release.RestartRequest{}, errRetryNoKey |
| 118 | } |
| 119 | signed, err := release.SignHostRetry(release.RootSigner(s.cfg.Key), acct, app, host, held.build, held.book, held.epoch, time.Now()) |
| 120 | if err != nil { |
| 121 | return release.RestartRequest{}, err |
| 122 | } |
| 123 | req, err := release.AppendHostRetry(ctx, s.cfg.Releases, acct, app, host, signed) |
| 124 | if err != nil { |
| 125 | return release.RestartRequest{}, err |
| 126 | } |
| 127 | s.bumpSerial(ctx) |
| 128 | return req, nil |
| 129 | } |
| 130 | |
| 131 | func (s *Server) handleAPIInstanceRetry(w http.ResponseWriter, r *http.Request) { |
| 132 | addr, acct, app, ok := s.apiAccountApp(w, r) |
| 133 | if !ok { |
| 134 | return |
| 135 | } |
| 136 | host := r.PathValue("host") |
| 137 | if !provision.ValidHostShape(host) { |
| 138 | http.NotFound(w, r) |
| 139 | return |
| 140 | } |
| 141 | p, _ := principalFrom(r.Context()) |
| 142 | if !s.allowWrite(p, time.Now()) { |
| 143 | jsonError(w, http.StatusTooManyRequests, "too many requests. wait a minute and try again") |
| 144 | return |
| 145 | } |
| 146 | if s.cfg.Releases == nil { |
| 147 | jsonError(w, http.StatusServiceUnavailable, |
| 148 | "no deployment bucket on this host. this console cannot request a retry") |
| 149 | return |
| 150 | } |
| 151 | if s.cfg.Key == nil { |
| 152 | jsonError(w, http.StatusServiceUnavailable, apiRetryNoKey) |
| 153 | return |
| 154 | } |
| 155 | var req struct { |
| 156 | Message string `json:"message"` |
| 157 | } |
| 158 | if !readJSON(w, r, &req) { |
| 159 | return |
| 160 | } |
| 161 | message, ok := checkedActivityMessage(w, req.Message) |
| 162 | if !ok { |
| 163 | return |
| 164 | } |
| 165 | ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second) |
| 166 | defer cancel() |
| 167 | held, err := s.retryHostBuild(ctx, acct, app, host) |
| 168 | switch { |
| 169 | case errors.Is(err, errRetryNoRecord): |
| 170 | http.NotFound(w, r) |
| 171 | return |
| 172 | case errors.Is(err, errRetryNotHeld): |
| 173 | jsonError(w, http.StatusConflict, fmt.Sprintf(apiRetryNotHeld, host)) |
| 174 | return |
| 175 | case errors.Is(err, errRetryUnsupported): |
| 176 | jsonError(w, http.StatusConflict, apiRetryUnsupported) |
| 177 | return |
| 178 | case errors.Is(err, errRetryUnsaved): |
| 179 | jsonError(w, http.StatusConflict, apiRetryUnsaved) |
| 180 | return |
| 181 | case err != nil: |
| 182 | log.Printf("console: api instance retry %s/%s/%s: %v", acct.Sqid, app, host, err) |
| 183 | jsonError(w, http.StatusBadGateway, "could not read the instance") |
| 184 | return |
| 185 | } |
| 186 | build := held.build |
| 187 | rr, err := s.requestHostRetry(ctx, acct.Sqid, app, host, held) |
| 188 | if err != nil { |
| 189 | log.Printf("console: api instance retry %s/%s/%s: %v", acct.Sqid, app, host, err) |
| 190 | jsonError(w, http.StatusBadGateway, apiRetryStore) |
| 191 | return |
| 192 | } |
| 193 | s.auditActivity(acct, app, addr, Activity{Action: "retry", TargetType: "instance", Target: host, |
| 194 | Message: message, Detail: fmt.Sprintf(apiRetryHostAudit, build, host, "cli")}) |
| 195 | writeJSON(w, http.StatusOK, map[string]any{"result": "requested", "id": rr.ID, "version": build, "host": host}) |
| 196 | } |
| 197 | |
amadan