| 1 | // The NARROWED restart signal (discussion #16): a request to cycle ONE route, |
| 2 | // or the routes reading ONE config environment, rather than every instance an |
| 3 | // app owns. |
| 4 | // |
| 5 | // Why a second object rather than a field on restart-gen.json. A box acts on |
| 6 | // that object's ETAG MOVING — it never reads the content — so a selector |
| 7 | // written INTO it would be invisible to every box running today's binary, and |
| 8 | // those boxes would read the movement the only way they know how: restart the |
| 9 | // whole app. For titogo that is nineteen production routes cycled to deliver |
| 10 | // one credential to one of them. The rule this repo already has for wire |
| 11 | // shapes ("anything written before your change must still parse") has a twin |
| 12 | // that this file is built on: an old reader that cannot understand a NARROWED |
| 13 | // request must do NOTHING, never something wider than was asked for. |
| 14 | // |
| 15 | // A sibling key gets that for free. Nothing that exists today reads |
| 16 | // restart-scope.json, so: |
| 17 | // |
| 18 | // - OLD box, NEW (narrowed) request — never looks at the key, sees no |
| 19 | // movement on restart-gen.json, cycles nothing. The request simply does |
| 20 | // not reach it. The console says which routes it addressed and on which |
| 21 | // box's registry it resolved them, so an operator whose fleet still has |
| 22 | // an old box can see that it is one box's answer. |
| 23 | // - NEW box, OLD (app-wide) request — restart-gen.json moves exactly as it |
| 24 | // always did and this file takes no part. |
| 25 | // - NEW box, FUTURE request kind — an entry whose `kind` this binary does |
| 26 | // not know is SKIPPED, not widened (see RestartScope.Selects). A selector |
| 27 | // nobody here understands addresses nothing. |
| 28 | // |
| 29 | // Bucket layout addition: |
| 30 | // |
| 31 | // apps/<acct>/<app>/restart-scope.json CAS-guarded, schema-1 request log |
| 32 | // |
| 33 | // It is a LOG rather than a single generation, which is the one place this |
| 34 | // shape costs more than restart-gen.json's. Movement is enough to carry "some |
| 35 | // restart was asked for"; it is not enough to carry WHICH routes, because two |
| 36 | // requests inside one 2s poll window collapse into one etag move and the |
| 37 | // earlier selector would be lost — silently, having told its caller |
| 38 | // "requested". Recording each request with an id, and remembering the last id |
| 39 | // acted on, makes a coalesced pair two requests again. |
| 40 | // |
| 41 | // The content is TRUSTED here in a way restart-gen.json's is not, so it is |
| 42 | // worth saying exactly how far that goes: an entry can only ever NARROW. The |
| 43 | // box matches entries against routes it already serves for that account and |
| 44 | // app — the account, the app and the route set come from the box's own |
| 45 | // registry, never from this object — so the worst a forged or corrupted entry |
| 46 | // can do is name a selector matching every route of one app, which is |
| 47 | // precisely what the unsigned app-wide object next to it already permits. |
| 48 | // Nothing about what runs, or about which account owns what, is decided here. |
| 49 | package release |
| 50 | |
| 51 | import ( |
| 52 | "context" |
| 53 | "crypto/rand" |
| 54 | "encoding/hex" |
| 55 | "encoding/json" |
| 56 | "errors" |
| 57 | "fmt" |
| 58 | "io" |
| 59 | "os" |
| 60 | "strings" |
| 61 | "time" |
| 62 | |
| 63 | "github.com/carlosframework/platform/internal/registry" |
| 64 | ) |
| 65 | |
| 66 | // The selector kinds. Only Host and Environment are ever written into the |
| 67 | // object; App names an app-wide request, which is restart-gen.json's job and |
| 68 | // travels no further than the HTTP door and the CLI that talks to it. It |
| 69 | // lives here so the three of them spell it the same way. |
| 70 | const ( |
| 71 | RestartScopeHost = "host" |
| 72 | RestartScopeEnvironment = "environment" |
| 73 | RestartScopeApp = "app" |
| 74 | |
| 75 | // RestartScopeRetry is `carlos instances retry <host>` (zero-downtime spec |
| 76 | // §8.4): Value is the host whose failure hold to clear. It is NOT a |
| 77 | // restart. Selects does not understand it, so a box built before it |
| 78 | // steps over it, and a box built after it hands it to the rollout manager |
| 79 | // instead of cycling anything (internal/host/restartscope.go). Unlike a |
| 80 | // restart, a retry lifts a restriction, so the file header's narrowing |
| 81 | // argument does not cover it: it acts only on a root-signed grant in |
| 82 | // Signed (HostRetryGrant, spec §11), and a box drops one without. |
| 83 | // |
| 84 | // It is the kind of every entry in the host-retry log (AppendHostRetry, |
| 85 | // hostretrylog.go), and no longer written into this one |
| 86 | // (ValidRestartScope): a console built before Signed strips it from every |
| 87 | // entry when it re-marshals this log. A box still verifies and applies a |
| 88 | // signed retry an earlier console left here, with the same checks. |
| 89 | RestartScopeRetry = "retry" |
| 90 | ) |
| 91 | |
| 92 | // restartScopeKeep bounds the log. Entries older than the last few dozen are |
| 93 | // dropped on the next append: a box that has been away long enough to miss |
| 94 | // them re-baselines (see PendingSince) rather than replaying a week of |
| 95 | // requests, and the object stays a small, cheap read. |
| 96 | const restartScopeKeep = 32 |
| 97 | |
| 98 | // RestartScopeNone is the mark a box records for "I have looked and there is |
| 99 | // nothing here" — the object is absent, or present with an empty log. The two |
| 100 | // collapse deliberately: an object that appears carrying its first request |
| 101 | // moves the mark from this sentinel to that request's id, which is movement, |
| 102 | // which is exactly what it is. |
| 103 | // |
| 104 | // The NUL byte is what makes it safe as a sentinel — no request id contains |
| 105 | // one, since ids are hex — and it is the same trick the box's own absent |
| 106 | // marker uses for etags. |
| 107 | const RestartScopeNone = "\x00none" |
| 108 | |
| 109 | // RestartScope is one selector: what to narrow a restart to. |
| 110 | // |
| 111 | // Kind is compared, never interpreted — an unknown one addresses nothing. |
| 112 | // Value is matched against a registry column and never becomes a bucket key, |
| 113 | // a filesystem path or a shell word; the strict shape checks belong at the |
| 114 | // door that accepts it from a person, where a typo can still be answered with |
| 115 | // what actually exists. |
| 116 | type RestartScope struct { |
| 117 | Kind string `json:"kind"` |
| 118 | Value string `json:"value"` |
| 119 | // Signed is a retry's root-signed grant (HostRetryGrant, spec §11): a |
| 120 | // box clears a hold only on a grant that verifies against this entry's |
| 121 | // account, app and host, so a box that can write this log cannot forge |
| 122 | // a retry (Astra impl-6 finding 5). Only a retry carries it, and only in |
| 123 | // the host-retry log is one written now (AppendHostRetry); a legacy |
| 124 | // entry in the restart-scope log may still carry one. It is additive: |
| 125 | // an older box's lax parse steps over it, and an unsigned retry is |
| 126 | // dropped by every box that checks. |
| 127 | Signed string `json:"signed,omitempty"` |
| 128 | } |
| 129 | |
| 130 | // RestartRequest is one recorded request. ID is what a box remembers; nothing |
| 131 | // is derived from it beyond "have I acted on this one". |
| 132 | type RestartRequest struct { |
| 133 | ID string `json:"id"` |
| 134 | RequestedAt string `json:"requested_at"` |
| 135 | Scope RestartScope `json:"scope"` |
| 136 | } |
| 137 | |
| 138 | // ScopedRestartGeneration is the object: apps/<acct>/<app>/restart-scope.json. |
| 139 | // Requests are OLDEST FIRST, so the newest is the last element and a box's |
| 140 | // "what have I not acted on" is a suffix. |
| 141 | type ScopedRestartGeneration struct { |
| 142 | Schema int `json:"schema"` // 1 |
| 143 | Requests []RestartRequest `json:"requests"` |
| 144 | } |
| 145 | |
| 146 | // RestartScopeKey is the bucket key of app's narrowed-request log, derived |
| 147 | // from AppPrefix like every other per-app key. |
| 148 | func RestartScopeKey(acct, app string) string { return AppPrefix(acct, app) + "restart-scope.json" } |
| 149 | |
| 150 | // Selects reports whether rt is addressed by this scope, and whether this |
| 151 | // binary understands the scope's kind at all. |
| 152 | // |
| 153 | // The second return is the forward-compatibility hinge, and the reason this |
| 154 | // is two bools rather than one. A selector kind added after this binary was |
| 155 | // built must address NOTHING here — not everything — so a caller that gets |
| 156 | // understood=false skips the whole request rather than falling back to the |
| 157 | // app. Widening on ignorance is the failure this whole file exists to avoid. |
| 158 | // |
| 159 | // The caller has already narrowed to the right account and app: this decides |
| 160 | // only which of THOSE routes the selector names. |
| 161 | func (sc RestartScope) Selects(rt registry.Route) (selected, understood bool) { |
| 162 | switch sc.Kind { |
| 163 | case RestartScopeHost: |
| 164 | return rt.Host == sc.Value, true |
| 165 | case RestartScopeEnvironment: |
| 166 | // An environment selector never matches the unbound routes: "" is not |
| 167 | // an environment name, it is the absence of one, and `--environment ""` |
| 168 | // cannot be typed. A request to cycle what reads the DEFAULT bundle is |
| 169 | // an app-wide request, and has the app-wide object. |
| 170 | return sc.Value != "" && rt.Environment == sc.Value, true |
| 171 | } |
| 172 | return false, false |
| 173 | } |
| 174 | |
| 175 | // ValidRestartScope refuses a selector this package would write but no box |
| 176 | // could act on. It is the last gate before the bucket, not the first: the |
| 177 | // door that took the value from a person has already checked its shape and |
| 178 | // answered a typo with what exists. |
| 179 | // |
| 180 | // A retry, and any signature, is refused: retries ride the host-retry log |
| 181 | // (AppendHostRetry), where no console older than it can strip a grant's |
| 182 | // signature by re-marshalling the log. |
| 183 | func ValidRestartScope(sc RestartScope) error { |
| 184 | switch sc.Kind { |
| 185 | case RestartScopeHost, RestartScopeEnvironment: |
| 186 | case RestartScopeRetry: |
| 187 | return fmt.Errorf("restart scope: a retry is written to the host-retry log, not this one") |
| 188 | default: |
| 189 | return fmt.Errorf("restart scope: unknown kind %q", sc.Kind) |
| 190 | } |
| 191 | if err := validScopeValue(sc.Kind, sc.Value); err != nil { |
| 192 | return err |
| 193 | } |
| 194 | if sc.Signed != "" { |
| 195 | return fmt.Errorf("restart scope: %s carries a signature, which only a retry may", sc.Kind) |
| 196 | } |
| 197 | return nil |
| 198 | } |
| 199 | |
| 200 | // validScopeValue is the shape check every entry's Value passes: present, |
| 201 | // at most a hostname's length, and no NUL or line break. |
| 202 | func validScopeValue(kind, value string) error { |
| 203 | if value == "" { |
| 204 | return fmt.Errorf("restart scope: %s has no value", kind) |
| 205 | } |
| 206 | if len(value) > 253 || strings.ContainsAny(value, "\x00\n\r") { |
| 207 | return fmt.Errorf("restart scope: %s value is not usable", kind) |
| 208 | } |
| 209 | return nil |
| 210 | } |
| 211 | |
| 212 | // LoadRestartScopes reads the log. found=false means the object is absent, |
| 213 | // which is the ordinary state of almost every app and is legal forever. |
| 214 | // |
| 215 | // A newer schema is an ERROR rather than a best-effort parse: a box that |
| 216 | // cannot be sure what a request means must do nothing, and its caller records |
| 217 | // nothing, so the request is picked up whenever the box is upgraded rather |
| 218 | // than dropped or guessed at. |
| 219 | func LoadRestartScopes(ctx context.Context, s Store, acct, app string) (*ScopedRestartGeneration, bool, error) { |
| 220 | key := RestartScopeKey(acct, app) |
| 221 | raw, _, err := s.Get(ctx, key) |
| 222 | if errors.Is(err, os.ErrNotExist) { |
| 223 | return &ScopedRestartGeneration{Schema: 1}, false, nil |
| 224 | } |
| 225 | if err != nil { |
| 226 | return nil, false, err |
| 227 | } |
| 228 | g, err := parseRestartScopes(key, raw) |
| 229 | if err != nil { |
| 230 | return nil, false, err |
| 231 | } |
| 232 | return g, true, nil |
| 233 | } |
| 234 | |
| 235 | func parseRestartScopes(key string, raw []byte) (*ScopedRestartGeneration, error) { |
| 236 | var g ScopedRestartGeneration |
| 237 | if err := json.Unmarshal(raw, &g); err != nil { |
| 238 | return nil, fmt.Errorf("restart scopes at %s: %w", key, err) |
| 239 | } |
| 240 | if g.Schema > 1 { |
| 241 | return nil, fmt.Errorf("restart scopes at %s: schema %d is newer than this binary understands — upgrade carlos", key, g.Schema) |
| 242 | } |
| 243 | return &g, nil |
| 244 | } |
| 245 | |
| 246 | // Newest is the id of the last request in the log, or RestartScopeNone for an |
| 247 | // empty one — the value a box records when it has nothing to act on. |
| 248 | func (g *ScopedRestartGeneration) Newest() string { |
| 249 | if g == nil || len(g.Requests) == 0 { |
| 250 | return RestartScopeNone |
| 251 | } |
| 252 | return g.Requests[len(g.Requests)-1].ID |
| 253 | } |
| 254 | |
| 255 | // PendingSince returns the requests recorded after mark — what a box that has |
| 256 | // already acted up to mark still owes — and whether mark could be located at |
| 257 | // all. |
| 258 | // |
| 259 | // found=false means the mark is an id that is no longer in the log: the box |
| 260 | // has been away long enough for its position to roll off the end. There is no |
| 261 | // honest answer to "what did I miss" then, and the two candidate guesses are |
| 262 | // not equal — replaying the whole log would cycle routes for requests served |
| 263 | // days ago, while re-baselining drops requests that are recoverable by typing |
| 264 | // the verb again. Callers re-baseline; see the box's restart pass, which makes |
| 265 | // the same choice everywhere else it is offered. |
| 266 | // |
| 267 | // RestartScopeNone as the mark is NOT an unknown position: it is a box that |
| 268 | // looked and found nothing, so everything now in the log is new to it. |
| 269 | func (g *ScopedRestartGeneration) PendingSince(mark string) (pending []RestartRequest, found bool) { |
| 270 | if g == nil { |
| 271 | return nil, true |
| 272 | } |
| 273 | if mark == RestartScopeNone { |
| 274 | return g.Requests, true |
| 275 | } |
| 276 | for i, r := range g.Requests { |
| 277 | if r.ID == mark { |
| 278 | return g.Requests[i+1:], true |
| 279 | } |
| 280 | } |
| 281 | if len(g.Requests) == 0 { |
| 282 | // The log emptied under a mark that named a request — only possible if |
| 283 | // the object was replaced wholesale. Unknown position, same answer. |
| 284 | return nil, false |
| 285 | } |
| 286 | return nil, false |
| 287 | } |
| 288 | |
| 289 | // AppendRestartScope records one narrowed request and returns it. |
| 290 | // |
| 291 | // A CAS conflict is RETRIED here, which is the opposite of TouchRestartGen's |
| 292 | // treatment of the same error, and the difference is the whole point of this |
| 293 | // object. There, any touch means "restart this app", so a racing writer has |
| 294 | // already fired the exact signal this call exists to fire and a retry would |
| 295 | // only manufacture a second restart. Here the other writer's request names a |
| 296 | // DIFFERENT set of routes: dropping ours would answer "requested" for a |
| 297 | // restart that will never happen. So we re-read and append again. |
| 298 | // |
| 299 | // The retry budget is small on purpose. This is a human-paced verb; a |
| 300 | // contended object means something is wrong that another lap will not fix, |
| 301 | // and the caller gets an error rather than a lie. |
| 302 | func claimRawAppendRestartScope(ctx context.Context, s Store, acct, app string, scope RestartScope) (RestartRequest, error) { |
| 303 | if !ValidAccount(acct) { |
| 304 | return RestartRequest{}, fmt.Errorf("restart: invalid account %q", acct) |
| 305 | } |
| 306 | if !ValidApp(app) { |
| 307 | return RestartRequest{}, fmt.Errorf("restart: invalid app %q", app) |
| 308 | } |
| 309 | if err := ValidRestartScope(scope); err != nil { |
| 310 | return RestartRequest{}, err |
| 311 | } |
| 312 | return claimRawAppendRequestLog(ctx, s, RestartScopeKey(acct, app), scope, "restart") |
| 313 | } |
| 314 | |
| 315 | // claimRawAppendRequestLog is the CAS append both request logs share (the |
| 316 | // restart-scope log and the host-retry log): a fresh random id, the log |
| 317 | // re-read and appended to on a conflict, at most restartScopeKeep entries |
| 318 | // kept, and a log this binary cannot parse refused rather than |
| 319 | // overwritten. verb names the request in the conflict error. The caller |
| 320 | // has validated the account, the app and the scope. |
| 321 | func claimRawAppendRequestLog(ctx context.Context, s Store, key string, scope RestartScope, verb string) (RestartRequest, error) { |
| 322 | id := make([]byte, 16) |
| 323 | if _, err := io.ReadFull(rand.Reader, id); err != nil { |
| 324 | return RestartRequest{}, err |
| 325 | } |
| 326 | req := RestartRequest{ |
| 327 | ID: hex.EncodeToString(id), |
| 328 | RequestedAt: time.Now().UTC().Format(time.RFC3339), |
| 329 | Scope: scope, |
| 330 | } |
| 331 | var lastErr error |
| 332 | for attempt := 0; attempt < 3; attempt++ { |
| 333 | raw, etag, err := s.Get(ctx, key) |
| 334 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 335 | return RestartRequest{}, err |
| 336 | } |
| 337 | g := &ScopedRestartGeneration{Schema: 1} |
| 338 | if err == nil { |
| 339 | if g, err = parseRestartScopes(key, raw); err != nil { |
| 340 | // Refusing beats overwriting: a log this binary cannot read is |
| 341 | // one whose entries it would silently drop on the next write. |
| 342 | return RestartRequest{}, err |
| 343 | } |
| 344 | } else { |
| 345 | etag = "" // create-only |
| 346 | } |
| 347 | g.Schema = 1 |
| 348 | g.Requests = append(g.Requests, req) |
| 349 | if len(g.Requests) > restartScopeKeep { |
| 350 | g.Requests = g.Requests[len(g.Requests)-restartScopeKeep:] |
| 351 | } |
| 352 | body, err := json.Marshal(g) |
| 353 | if err != nil { |
| 354 | return RestartRequest{}, err |
| 355 | } |
| 356 | if _, err := s.Put(ctx, key, body, etag); err != nil { |
| 357 | if errors.Is(err, ErrCASConflict) { |
| 358 | lastErr = err |
| 359 | continue |
| 360 | } |
| 361 | return RestartRequest{}, err |
| 362 | } |
| 363 | return req, nil |
| 364 | } |
| 365 | return RestartRequest{}, fmt.Errorf("%s: %s is being written by someone else — nothing was requested; try again: %w", verb, key, lastErr) |
| 366 | } |
| 367 | func AppendRestartScope(ctx context.Context, s Store, acct, app string, scope RestartScope) (claimResult0 RestartRequest, claimResult1 error) { |
| 368 | claimResult1 = WithAppMutation(ctx, s, acct, app, "release.AppendRestartScope", func(claimContext context.Context) error { |
| 369 | claimResult0, claimResult1 = claimRawAppendRestartScope(claimContext, s, acct, app, scope) |
| 370 | return claimResult1 |
| 371 | }) |
| 372 | return |
| 373 | } |
| 374 | |
amadan