| 1 | package activator |
| 2 | |
| 3 | // The instance key (region HA request L21; rastrillo/rastrillo #66). An app |
| 4 | // on CARLOS seals its data with a key. Before this file the app minted that |
| 5 | // key itself after it started, so a box that died before the key reached a |
| 6 | // backup left a database nothing could unseal. The platform now mints the |
| 7 | // key on the box, into the instance's own <host>.env as |
| 8 | // CARLOS_INSTANCE_KEY, before the app's first start, and never rewrites it. |
| 9 | // rastrillo's carlos.InstanceKey reads it verbatim when the app has no key |
| 10 | // of its own. |
| 11 | // |
| 12 | // The rules, each a ruling in |
| 13 | // docs/superpowers/plans/2026-10-10-instance-key-provision.md: |
| 14 | // - create-only: a key already in the file is never rewritten, whatever |
| 15 | // its shape. A malformed one is reported and left in place, as |
| 16 | // EnsureAdminToken does with a malformed token; |
| 17 | // - the decision is made on the effective env (instancekey.Effective, |
| 18 | // the rule mergeEnv applies): a key the rendered configuration |
| 19 | // supplies is the app's key and nothing is minted; an empty own value |
| 20 | // is no key; |
| 21 | // - nothing is minted until the row's configuration is here. The files |
| 22 | // say what this box has delivered, not what the configuration sets: a |
| 23 | // first wake can land before the config sync's first render, and a |
| 24 | // key minted then would shadow the configured one for good once it |
| 25 | // arrived. So before it mints, the ensure asks the bucket |
| 26 | // (ExecBackend.ConfiguredKey, the replica gate's reader) whether the |
| 27 | // configuration sets a key now; if it does, or the bucket cannot say, |
| 28 | // the launch is deferred (ErrInstanceKeyDeferred) and retried once the |
| 29 | // sync has delivered the file; |
| 30 | // - durable every time: before an ensure reports success the file is |
| 31 | // 0600 (fchmod on its descriptor), the file is fsynced and so is its |
| 32 | // directory, when the key was already there too. A sync that failed |
| 33 | // once is redone by the next ensure, never skipped because the bytes |
| 34 | // are already there. The known limit: once fsync has returned EIO, |
| 35 | // Linux may mark the pages clean, so a later fsync's success does not |
| 36 | // prove the bytes reached the disk. The retry narrows the gap; it |
| 37 | // cannot close it, and no create-only design can; |
| 38 | // - a mint that fails part way through its append is undone: the file |
| 39 | // is truncated back to its length before the append and fsynced, so a |
| 40 | // short key line never survives to be read as present. Those bytes |
| 41 | // were never reported, so removing them rewrites no key; |
| 42 | // - the value is never logged, never put in an error, and never leaves |
| 43 | // the file except in the launch environment; |
| 44 | // - a flat row's own file is opened O_NOFOLLOW on every path, read and |
| 45 | // append alike (openOwnEnvNoFollow): a symlink there is refused, never |
| 46 | // followed into a file a same-uid sibling chose. A nested row's file |
| 47 | // gets the same from instancefs. |
| 48 | |
| 49 | import ( |
| 50 | "context" |
| 51 | "crypto/rand" |
| 52 | "encoding/base64" |
| 53 | "errors" |
| 54 | "fmt" |
| 55 | "io" |
| 56 | "os" |
| 57 | "path/filepath" |
| 58 | "strings" |
| 59 | "time" |
| 60 | |
| 61 | "github.com/carlosframework/platform/internal/instancefs" |
| 62 | "github.com/carlosframework/platform/internal/instancekey" |
| 63 | "github.com/carlosframework/platform/internal/registry" |
| 64 | ) |
| 65 | |
| 66 | // InstanceKeyVar is the variable the key is delivered as. |
| 67 | const InstanceKeyVar = instancekey.Var |
| 68 | |
| 69 | // instanceKeyBytes is the key's length before encoding: 32 CSPRNG bytes, |
| 70 | // standard base64 with padding, 44 characters. rastrillo uses the exact |
| 71 | // bytes the env file yields, so this encoding is fixed for the life of |
| 72 | // every key minted with it; changing it is a rotation. |
| 73 | const instanceKeyBytes = 32 |
| 74 | |
| 75 | // InstanceKeyOutcome is what one ensure did. Its zero value is |
| 76 | // InstanceKeyFailed, what every ensure answers beside an error, so a |
| 77 | // caller that reads the outcome before the error can never count a |
| 78 | // refused row as anything. |
| 79 | type InstanceKeyOutcome int |
| 80 | |
| 81 | const ( |
| 82 | // InstanceKeyFailed: the ensure returned an error and reports nothing. |
| 83 | // It refused the row (fenced, conflicting, a configuration not yet |
| 84 | // delivered) or failed part way (an open, a write, a sync); the error |
| 85 | // says which. Nothing it may have written is reported as durable. |
| 86 | InstanceKeyFailed InstanceKeyOutcome = iota |
| 87 | // InstanceKeyPresent: the own env file already carried a key. No byte |
| 88 | // was written; the file was made durable. |
| 89 | InstanceKeyPresent |
| 90 | // InstanceKeyMinted: a key was appended and made durable. |
| 91 | InstanceKeyMinted |
| 92 | // InstanceKeyFromConfig: the rendered configuration supplies the key. |
| 93 | // Nothing was written. |
| 94 | InstanceKeyFromConfig |
| 95 | // InstanceKeyNoEnvFile: the backfill found no own env file, so the host |
| 96 | // has never been started here. Nothing was written; its first start |
| 97 | // mints the key. |
| 98 | InstanceKeyNoEnvFile |
| 99 | // InstanceKeyReplica: a replica row (#162). A replica is never minted |
| 100 | // a key: it may hold only its home's (ruling 26, replicaKeyGate). |
| 101 | InstanceKeyReplica |
| 102 | ) |
| 103 | |
| 104 | func (o InstanceKeyOutcome) String() string { |
| 105 | switch o { |
| 106 | case InstanceKeyFailed: |
| 107 | return "failed" |
| 108 | case InstanceKeyPresent: |
| 109 | return "present" |
| 110 | case InstanceKeyMinted: |
| 111 | return "minted" |
| 112 | case InstanceKeyFromConfig: |
| 113 | return "from-config" |
| 114 | case InstanceKeyNoEnvFile: |
| 115 | return "no-env-file" |
| 116 | case InstanceKeyReplica: |
| 117 | return "replica" |
| 118 | } |
| 119 | return fmt.Sprintf("outcome(%d)", int(o)) |
| 120 | } |
| 121 | |
| 122 | // instanceKeySync is the durability step's two syncs, a seam so a test can |
| 123 | // fail each one. Read and replaced only under adminTokenMu. |
| 124 | var instanceKeySync = struct { |
| 125 | file func(*os.File) error |
| 126 | dir func(string) error |
| 127 | }{file: (*os.File).Sync, dir: syncDir} |
| 128 | |
| 129 | // syncDir fsyncs a directory, so an entry created in it survives a power |
| 130 | // cut. |
| 131 | func syncDir(dir string) error { |
| 132 | d, err := os.Open(dir) |
| 133 | if err != nil { |
| 134 | return err |
| 135 | } |
| 136 | err = d.Sync() |
| 137 | if cerr := d.Close(); err == nil { |
| 138 | err = cerr |
| 139 | } |
| 140 | return err |
| 141 | } |
| 142 | |
| 143 | // instanceKeyWrite is the mint's append, a seam so a test can make it fail |
| 144 | // part way (a short write, ENOSPC). Read and replaced only under |
| 145 | // adminTokenMu. |
| 146 | var instanceKeyWrite = func(f *os.File, b []byte) (int, error) { return f.Write(b) } |
| 147 | |
| 148 | // instanceKeyHooks are the three test seams on the launch path's hot path, |
| 149 | // all nil in production; the sync and write seams above are what the |
| 150 | // durability tests need and stay on their own. beforeOpen runs under |
| 151 | // adminTokenMu after the read and the decision and before the ensure |
| 152 | // opens the file to append or settle, so a test can change the file |
| 153 | // between the two and pin the open's own symlink refusal, not only the |
| 154 | // read's. betweenPhases runs after the ensure's first read (and its |
| 155 | // bucket read, if any) and before it takes adminTokenMu, so a test can |
| 156 | // change the files between the two phases as a concurrent writer can. |
| 157 | // replicaGate runs at the start of every replica gate, after the env was |
| 158 | // read, so a test can change the env files between the read and the |
| 159 | // gate, as a concurrent config sync can. |
| 160 | var instanceKeyHooks struct { |
| 161 | beforeOpen func(path string) |
| 162 | betweenPhases func(path string) |
| 163 | replicaGate func() |
| 164 | } |
| 165 | |
| 166 | func mintInstanceKey() (string, error) { |
| 167 | raw := make([]byte, instanceKeyBytes) |
| 168 | if _, err := rand.Read(raw); err != nil { |
| 169 | return "", err |
| 170 | } |
| 171 | return base64.StdEncoding.EncodeToString(raw), nil |
| 172 | } |
| 173 | |
| 174 | // wellFormedInstanceKey reports whether key has the shape this file mints. |
| 175 | // A key that fails it is not necessarily broken (an operator may have |
| 176 | // written one by hand), which is why a failure is a warning, never a |
| 177 | // rotation. |
| 178 | func wellFormedInstanceKey(key string) bool { |
| 179 | if len(key) != base64.StdEncoding.EncodedLen(instanceKeyBytes) { |
| 180 | return false |
| 181 | } |
| 182 | raw, err := base64.StdEncoding.Strict().DecodeString(key) |
| 183 | return err == nil && len(raw) == instanceKeyBytes |
| 184 | } |
| 185 | |
| 186 | // configEnvName is the rendered configuration's file name beside an own env |
| 187 | // file: <host>.env's sibling <host>.config.env, the pair pathsFor names. |
| 188 | func configEnvName(envName string) string { |
| 189 | return strings.TrimSuffix(envName, ".env") + ".config.env" |
| 190 | } |
| 191 | |
| 192 | // ErrInstanceKeyDeferred is a first start held back because the row's |
| 193 | // configuration is not here yet: the bucket's configuration sets |
| 194 | // CARLOS_INSTANCE_KEY and this box has not delivered it, or the bucket |
| 195 | // could not be read to tell. Nothing is minted (ruling 5: a configured key |
| 196 | // is the app's, and one minted while it was on its way would shadow it for |
| 197 | // good). The launch fails and is retried: the config sync's next pass |
| 198 | // delivers the file, and the next start finds the key configured. |
| 199 | var ErrInstanceKeyDeferred = errors.New("activator: instance key not minted: the instance's configuration is not delivered") |
| 200 | |
| 201 | // configReadTimeout bounds each read of the bucket's configuration, by |
| 202 | // the mint and by the replica gate. |
| 203 | const configReadTimeout = 10 * time.Second |
| 204 | |
| 205 | // configuredKeyReader answers, for one row, the SHA-256 of the |
| 206 | // CARLOS_INSTANCE_KEY the bucket's configuration renders now, "" for |
| 207 | // none. nil is a box with no configuration source. |
| 208 | type configuredKeyReader func(context.Context) (string, error) |
| 209 | |
| 210 | // configuredKeyFor is b.ConfiguredKey for rt, each read bounded by |
| 211 | // configReadTimeout; nil when the backend has no reader. |
| 212 | func (b *ExecBackend) configuredKeyFor(rt registry.Route) configuredKeyReader { |
| 213 | if b.ConfiguredKey == nil { |
| 214 | return nil |
| 215 | } |
| 216 | return func(ctx context.Context) (string, error) { |
| 217 | ctx, cancel := context.WithTimeout(ctx, configReadTimeout) |
| 218 | defer cancel() |
| 219 | return b.ConfiguredKey(ctx, rt) |
| 220 | } |
| 221 | } |
| 222 | |
| 223 | // ensureInstanceKeyNoConfig makes sure path (an instance's own <host>.env |
| 224 | // on the flat layout) carries CARLOS_INSTANCE_KEY, creating the file if |
| 225 | // needed, for a row whose configuration can set no key: there is no |
| 226 | // reader, so nothing is asked before the mint. Tests use it; every launch |
| 227 | // path goes through ensureInstanceKeyFor, which asks the bucket first. |
| 228 | func ensureInstanceKeyNoConfig(path string) (InstanceKeyOutcome, error) { |
| 229 | return ensureInstanceKey(context.Background(), filepath.Dir(path), filepath.Base(path), false, true, nil) |
| 230 | } |
| 231 | |
| 232 | // ensureInstanceKeyBeneathNoConfig is ensureInstanceKeyNoConfig for a |
| 233 | // nested row's own env file, read and written through instancefs because |
| 234 | // the directory is tenant-writable (isolated-exec design §3.2). |
| 235 | func ensureInstanceKeyBeneathNoConfig(dir, name string) (InstanceKeyOutcome, error) { |
| 236 | return ensureInstanceKey(context.Background(), dir, name, true, true, nil) |
| 237 | } |
| 238 | |
| 239 | // ensureInstanceKeyFor is the launch paths' ensure for rt: none for a |
| 240 | // replica row, the nested body for a nested row, the flat one otherwise. |
| 241 | // ctx bounds the one bucket read a mint makes; a launch with none passes |
| 242 | // context.Background(). |
| 243 | // |
| 244 | // A fenced or conflicting row is refused and nothing is written, as |
| 245 | // BackfillInstanceKey refuses it. Every launch dispatcher (Start, |
| 246 | // StartGen, StartStaged, Seed) refuses those rows already, but on its own |
| 247 | // classification: the launch body classifies again in pathsFor, and p is |
| 248 | // that second answer. A layout journal written between the two (a |
| 249 | // migration starting mid-wake) makes p fenced, with the flat directory as |
| 250 | // its dataDir, and minting there would write to a half-migrated host |
| 251 | // (isolated-exec design §8.2). The refusal is decided on p, the paths the |
| 252 | // mint would use. Every caller names the host around the error ("instance |
| 253 | // key for <host>: ..."), so the refusal does not. |
| 254 | func (b *ExecBackend) ensureInstanceKeyFor(ctx context.Context, rt registry.Route, p instancePaths) (InstanceKeyOutcome, error) { |
| 255 | return b.ensureInstanceKeyAt(ctx, rt, p, true, "minted") |
| 256 | } |
| 257 | |
| 258 | // BackfillInstanceKey is the startup backfill's step for one row: a key is |
| 259 | // appended only to an own env file that already exists. A host with no file |
| 260 | // has never been started on this box, and its first start mints the key; |
| 261 | // creating a file here could also occupy the path a move's install expects |
| 262 | // to be empty (moveslandinstall.go placeStaged). A row this backend would |
| 263 | // refuse to start (fenced, conflict) is an error and nothing is written. |
| 264 | // The agent's backfill names the host when it logs the error, so the |
| 265 | // refusal does not. |
| 266 | func (b *ExecBackend) BackfillInstanceKey(ctx context.Context, rt registry.Route) (InstanceKeyOutcome, error) { |
| 267 | return b.ensureInstanceKeyAt(ctx, rt, b.pathsFor(rt), false, "backfilled") |
| 268 | } |
| 269 | |
| 270 | // ensureInstanceKeyAt is the two entry points' one guard: a replica row is |
| 271 | // never minted; a fenced or conflicting row is refused on p, the paths the |
| 272 | // mint would use, with verb naming which caller refused ("not minted", |
| 273 | // "not backfilled"); every other row is ensureInstanceKey on p's layout, |
| 274 | // with the bucket reader for rt. |
| 275 | func (b *ExecBackend) ensureInstanceKeyAt(ctx context.Context, rt registry.Route, p instancePaths, create bool, verb string) (InstanceKeyOutcome, error) { |
| 276 | if rt.Replica { |
| 277 | return InstanceKeyReplica, nil |
| 278 | } |
| 279 | switch p.layout { |
| 280 | case LayoutFenced, LayoutConflict: |
| 281 | return InstanceKeyFailed, fmt.Errorf("not %s: %s", verb, p.reason) |
| 282 | } |
| 283 | return ensureInstanceKey(ctx, p.dataDir, rt.Host+".env", p.layout == LayoutNested, create, b.configuredKeyFor(rt)) |
| 284 | } |
| 285 | |
| 286 | // openOwnEnv opens an own env file, <host>.env under dir: through |
| 287 | // instancefs when beneath (a nested row's tenant-writable directory), else |
| 288 | // through openOwnEnvNoFollow. Both refuse a link or a FIFO in the open. |
| 289 | // The one place the two layouts' opens meet. |
| 290 | func openOwnEnv(dir, name string, beneath bool, flags int, perm os.FileMode) (*os.File, error) { |
| 291 | if beneath { |
| 292 | return instancefs.OpenBeneath(dir, name, flags, perm) |
| 293 | } |
| 294 | return openOwnEnvNoFollow(filepath.Join(dir, name), flags, perm) |
| 295 | } |
| 296 | |
| 297 | // openInstanceEnv opens an own env file for append. Only a create passes |
| 298 | // a mode: openat2 refuses a non-zero mode without O_CREAT (EINVAL), and an |
| 299 | // existing file's mode is set by the fchmod that follows anyway. |
| 300 | func openInstanceEnv(dir, name string, beneath, create bool) (*os.File, error) { |
| 301 | flags, perm := os.O_APPEND|os.O_WRONLY, os.FileMode(0) |
| 302 | if create { |
| 303 | flags, perm = flags|os.O_CREATE, 0o600 |
| 304 | } |
| 305 | return openOwnEnv(dir, name, beneath, flags, perm) |
| 306 | } |
| 307 | |
| 308 | // errOwnEnvSymlink is a flat row's own env file found to be a symlink. The |
| 309 | // ensure refuses it on every path (mint, existing key, backfill, the claim |
| 310 | // settle): following it would append to, chmod and fsync a file a same-uid |
| 311 | // sibling chose (spec §7.2). |
| 312 | var errOwnEnvSymlink = errors.New("activator: the instance env file is a symlink, refused") |
| 313 | |
| 314 | // errOwnEnvNotRegular is a flat row's own env file, or its rendered |
| 315 | // configuration, found to be something other than a regular file: a FIFO, |
| 316 | // which a blocking open would wait on for a writer that never comes (the |
| 317 | // startup backfill holds adminTokenMu while it reads, so one planted FIFO |
| 318 | // would keep every instance on the box from waking), or a device. Refused |
| 319 | // on every path, as the link is. |
| 320 | var errOwnEnvNotRegular = errors.New("activator: the instance env file is not a regular file, refused") |
| 321 | |
| 322 | // ownEnvReadLimit caps a flat read of an own env file or its rendered |
| 323 | // configuration, as instancefs.ReadBeneathLimit caps the nested one: no |
| 324 | // read under adminTokenMu is unbounded. It is instancekey's one cap, the |
| 325 | // one relocation's reads of the same files use. |
| 326 | const ownEnvReadLimit = instancekey.EnvReadLimit |
| 327 | |
| 328 | // errOwnEnvTooLarge is a flat env file over ownEnvReadLimit. |
| 329 | var errOwnEnvTooLarge = errors.New("activator: the instance env file is larger than the limit, refused") |
| 330 | |
| 331 | // readOwnEnv reads a flat row's own env file, or its rendered |
| 332 | // configuration, through openOwnEnvNoFollow, so the read refuses a link |
| 333 | // or a FIFO exactly as the append does, and never reads more than |
| 334 | // ownEnvReadLimit. |
| 335 | func readOwnEnv(path string) ([]byte, error) { |
| 336 | f, err := openOwnEnvNoFollow(path, os.O_RDONLY, 0) |
| 337 | if err != nil { |
| 338 | return nil, err |
| 339 | } |
| 340 | defer f.Close() |
| 341 | raw, err := io.ReadAll(io.LimitReader(f, ownEnvReadLimit+1)) |
| 342 | if err != nil { |
| 343 | return nil, err |
| 344 | } |
| 345 | if len(raw) > ownEnvReadLimit { |
| 346 | return nil, errOwnEnvTooLarge |
| 347 | } |
| 348 | return raw, nil |
| 349 | } |
| 350 | |
| 351 | // settleInstanceEnv is the durability step every successful ensure ends |
| 352 | // with: 0600 on the descriptor, then syncInstanceEnv. Caller holds |
| 353 | // adminTokenMu. |
| 354 | func settleInstanceEnv(f *os.File, dir, path string) error { |
| 355 | if err := f.Chmod(0o600); err != nil { |
| 356 | return fmt.Errorf("chmod %s: %w", path, err) |
| 357 | } |
| 358 | return syncInstanceEnv(f, dir, path) |
| 359 | } |
| 360 | |
| 361 | // syncInstanceEnv fsyncs an own env file, then its directory: the two |
| 362 | // syncs every durability step ends with, the claim's settle included. |
| 363 | // Caller holds adminTokenMu. |
| 364 | func syncInstanceEnv(f *os.File, dir, path string) error { |
| 365 | if err := instanceKeySync.file(f); err != nil { |
| 366 | return fmt.Errorf("sync %s: %w", path, err) |
| 367 | } |
| 368 | if err := instanceKeySync.dir(dir); err != nil { |
| 369 | return fmt.Errorf("sync %s: %w", dir, err) |
| 370 | } |
| 371 | return nil |
| 372 | } |
| 373 | |
| 374 | // readInstanceEnvPair reads an own env file and its rendered configuration |
| 375 | // the same guarded way on both layouts: through instancefs on the nested |
| 376 | // one, and on the flat one through readOwnEnv, which refuses a link or a |
| 377 | // FIFO at either name and caps the read. exists is false when the own file |
| 378 | // is absent (raw nil); an absent configuration is nil cfg and no error. |
| 379 | func readInstanceEnvPair(dir, name string, beneath bool) (raw, cfg []byte, exists bool, err error) { |
| 380 | read := func(n string) ([]byte, error) { |
| 381 | if beneath { |
| 382 | return instancefs.ReadBeneathLimit(dir, n, ownEnvReadLimit) |
| 383 | } |
| 384 | return readOwnEnv(filepath.Join(dir, n)) |
| 385 | } |
| 386 | raw, err = read(name) |
| 387 | switch { |
| 388 | case errors.Is(err, os.ErrNotExist): |
| 389 | raw = nil |
| 390 | case err != nil: |
| 391 | return nil, nil, false, fmt.Errorf("read %s: %w", filepath.Join(dir, name), err) |
| 392 | default: |
| 393 | exists = true |
| 394 | } |
| 395 | cfg, err = read(configEnvName(name)) |
| 396 | if err != nil && !errors.Is(err, os.ErrNotExist) { |
| 397 | return nil, nil, false, fmt.Errorf("read %s: %w", filepath.Join(dir, configEnvName(name)), err) |
| 398 | } |
| 399 | return raw, cfg, exists, nil |
| 400 | } |
| 401 | |
| 402 | // bucketVerdict is what the bucket said before the lock was taken: asked |
| 403 | // is whether a mint looked next on that read and the bucket was consulted; |
| 404 | // deferral is its answer when the mint must wait (a configured key not yet |
| 405 | // delivered, or a read that failed), nil when it may go ahead. |
| 406 | type bucketVerdict struct { |
| 407 | asked bool |
| 408 | deferral error |
| 409 | } |
| 410 | |
| 411 | // ensureInstanceKey is the shared body. It holds adminTokenMu, the lock |
| 412 | // every mint into <host>.env takes, so a key append and a token append can |
| 413 | // never interleave. create false is the backfill: an absent file is |
| 414 | // InstanceKeyNoEnvFile and nothing is created. |
| 415 | // |
| 416 | // The bucket is read in two phases, so a slow or unreachable bucket for |
| 417 | // one row never holds the lock every other row's mint waits on. Outside |
| 418 | // the lock: read the files, and if a mint looks next, ask the bucket |
| 419 | // (configured; the one read a row ever costs). Under the lock: read the |
| 420 | // files again and decide from that read alone. A key that appeared in |
| 421 | // either file since is present or configured, and nothing is minted; a |
| 422 | // mint goes ahead only when the locked read still finds no key and the |
| 423 | // bucket was asked and said none. A mint that looks next under the lock |
| 424 | // without a verdict (the first read saw a key that has since gone) defers |
| 425 | // rather than mint on nothing, and the next attempt asks. Create-only, |
| 426 | // fail-closed and defer-on-unknown hold on every path. |
| 427 | func ensureInstanceKey(ctx context.Context, dir, name string, beneath, create bool, configured configuredKeyReader) (InstanceKeyOutcome, error) { |
| 428 | path := filepath.Join(dir, name) |
| 429 | // Phase one, outside the lock. Nothing delivered here sets a key. The |
| 430 | // configuration may still: a first wake can run before the config |
| 431 | // sync's first render of <host>.config.env (the fast-converge pass |
| 432 | // reconciles the row ready and syncs config last), and a key minted |
| 433 | // in that gap would shadow the configured one for good once it |
| 434 | // arrived (Effective: the own file wins). So ask the bucket whether |
| 435 | // the configuration sets a key now. One it sets is not here yet: |
| 436 | // defer, and the next start finds it delivered. A bucket that cannot |
| 437 | // be read cannot say, and defers the same way. With no reader there |
| 438 | // is no configuration source, and nothing can be on its way. A read |
| 439 | // error here is left for the locked read to report. |
| 440 | var pre bucketVerdict |
| 441 | if configured != nil { |
| 442 | if raw, cfg, exists, err := readInstanceEnvPair(dir, name, beneath); err == nil && (exists || create) { |
| 443 | if _, layer := instancekey.Effective(parseEnvBytes(cfg), parseEnvBytes(raw)); layer == instancekey.LayerNone { |
| 444 | pre.asked = true |
| 445 | digest, err := configured(ctx) |
| 446 | switch { |
| 447 | case err != nil: |
| 448 | pre.deferral = fmt.Errorf("%w: its configuration could not be read: %w", ErrInstanceKeyDeferred, err) |
| 449 | case digest != "": |
| 450 | pre.deferral = fmt.Errorf("%w: its configuration sets %s and this box has not delivered it yet (the next config sync does, and the next start finds it)", ErrInstanceKeyDeferred, InstanceKeyVar) |
| 451 | } |
| 452 | } |
| 453 | } |
| 454 | } |
| 455 | if instanceKeyHooks.betweenPhases != nil { |
| 456 | instanceKeyHooks.betweenPhases(path) |
| 457 | } |
| 458 | // Phase two, under the lock: the decision, from this read only. |
| 459 | adminTokenMu.Lock() |
| 460 | defer adminTokenMu.Unlock() |
| 461 | raw, cfg, exists, err := readInstanceEnvPair(dir, name, beneath) |
| 462 | if err != nil { |
| 463 | return InstanceKeyFailed, err |
| 464 | } |
| 465 | if !exists && !create { |
| 466 | return InstanceKeyNoEnvFile, nil |
| 467 | } |
| 468 | // The decision is the effective env's: the key the launch will hand |
| 469 | // the app, by the same rule mergeEnv applies. |
| 470 | key, layer := instancekey.Effective(parseEnvBytes(cfg), parseEnvBytes(raw)) |
| 471 | switch layer { |
| 472 | case instancekey.LayerConfig: |
| 473 | logPerFile("key-config", path, fmt.Sprintf("activator: %s is set by the app's configuration for %s: the platform mints none while it is", InstanceKeyVar, path)) |
| 474 | return InstanceKeyFromConfig, nil |
| 475 | case instancekey.LayerOwn: |
| 476 | if !wellFormedInstanceKey(key) { |
| 477 | logPerFile("key-shape", path, fmt.Sprintf("activator: %s in %s is not the shape the platform mints (44 base64 characters, 32 bytes): left in place, because the app may already have sealed data with it", InstanceKeyVar, path)) |
| 478 | } |
| 479 | // Present is not a no-op: a mint whose sync failed left these |
| 480 | // bytes behind, and this call is the retry that makes them |
| 481 | // durable before anything is started with them. |
| 482 | if instanceKeyHooks.beforeOpen != nil { |
| 483 | instanceKeyHooks.beforeOpen(path) |
| 484 | } |
| 485 | f, err := openInstanceEnv(dir, name, beneath, false) |
| 486 | if err != nil { |
| 487 | return InstanceKeyFailed, fmt.Errorf("open %s: %w", path, err) |
| 488 | } |
| 489 | defer f.Close() |
| 490 | if err := settleInstanceEnv(f, dir, path); err != nil { |
| 491 | return InstanceKeyFailed, err |
| 492 | } |
| 493 | return InstanceKeyPresent, nil |
| 494 | } |
| 495 | // A mint is next. It goes ahead only on the bucket's word from phase |
| 496 | // one; without it (the files changed between the two reads) nothing |
| 497 | // is known, and the mint defers rather than go on nothing. |
| 498 | if configured != nil { |
| 499 | switch { |
| 500 | case !pre.asked: |
| 501 | return InstanceKeyFailed, fmt.Errorf("%w: its env files changed while the configuration was being read (retried at the next start)", ErrInstanceKeyDeferred) |
| 502 | case pre.deferral != nil: |
| 503 | return InstanceKeyFailed, pre.deferral |
| 504 | } |
| 505 | } |
| 506 | minted, err := mintInstanceKey() |
| 507 | if err != nil { |
| 508 | return InstanceKeyFailed, fmt.Errorf("mint instance key for %s: %w", path, err) |
| 509 | } |
| 510 | if instanceKeyHooks.beforeOpen != nil { |
| 511 | instanceKeyHooks.beforeOpen(path) |
| 512 | } |
| 513 | f, err := openInstanceEnv(dir, name, beneath, create) |
| 514 | if errors.Is(err, os.ErrNotExist) && !create { |
| 515 | return InstanceKeyNoEnvFile, nil // removed since the read: nothing to backfill |
| 516 | } |
| 517 | if err != nil { |
| 518 | return InstanceKeyFailed, fmt.Errorf("open %s: %w", path, err) |
| 519 | } |
| 520 | defer f.Close() |
| 521 | // The length the append starts from, taken from the open descriptor: |
| 522 | // what a failed append is undone to. Under adminTokenMu it is |
| 523 | // len(raw); reading it here also keeps any byte another process |
| 524 | // appended since the read. |
| 525 | st, err := f.Stat() |
| 526 | if err != nil { |
| 527 | return InstanceKeyFailed, fmt.Errorf("stat %s: %w", path, err) |
| 528 | } |
| 529 | prefix := "" |
| 530 | if len(raw) > 0 && raw[len(raw)-1] != '\n' { |
| 531 | prefix = "\n" |
| 532 | } |
| 533 | line := []byte(prefix + InstanceKeyVar + "=" + minted + "\n") |
| 534 | if n, err := instanceKeyWrite(f, line); err != nil || n != len(line) { |
| 535 | if err == nil { |
| 536 | err = io.ErrShortWrite |
| 537 | } |
| 538 | return InstanceKeyFailed, undoInstanceKeyAppend(f, st.Size(), path, err) |
| 539 | } |
| 540 | if err := settleInstanceEnv(f, dir, path); err != nil { |
| 541 | return InstanceKeyFailed, err |
| 542 | } |
| 543 | return InstanceKeyMinted, nil |
| 544 | } |
| 545 | |
| 546 | // undoInstanceKeyAppend removes a failed append's bytes: it truncates f |
| 547 | // back to size, the length before the append, and fsyncs, so a short key |
| 548 | // line can never survive to be read as present. Nothing past size was |
| 549 | // ever reported to anyone. Returns the write error, joined with any |
| 550 | // failure to undo it. Caller holds adminTokenMu. |
| 551 | func undoInstanceKeyAppend(f *os.File, size int64, path string, werr error) error { |
| 552 | err := fmt.Errorf("write %s: %w", path, werr) |
| 553 | if terr := f.Truncate(size); terr != nil { |
| 554 | return errors.Join(err, fmt.Errorf("undo the partial write to %s: %w", path, terr)) |
| 555 | } |
| 556 | if serr := instanceKeySync.file(f); serr != nil { |
| 557 | return errors.Join(err, fmt.Errorf("sync %s after undoing the partial write: %w", path, serr)) |
| 558 | } |
| 559 | return err |
| 560 | } |
| 561 | |
| 562 | // errInstanceEnvNeedsWrite is settleClaimInstanceEnv's answer for a file |
| 563 | // whose mode is not 0600: making it so is a write, which the claim path |
| 564 | // does only under its guard. |
| 565 | var errInstanceEnvNeedsWrite = errors.New("activator: the instance env file needs its mode set") |
| 566 | |
| 567 | // settleClaimInstanceEnv is the claim launch source's durability step for |
| 568 | // an env file that already carries what the launch needs: it makes the |
| 569 | // file and its directory durable and writes nothing. fsync changes no |
| 570 | // byte, mode or name, so it is not a host write and runs without the |
| 571 | // claim's guard. A file that is not 0600 is errInstanceEnvNeedsWrite, and |
| 572 | // the caller takes the guarded path, whose ensure sets the mode when the |
| 573 | // own file holds the key (and leaves it when the key is configured or the |
| 574 | // row is a replica: EnsureClaimAdminToken). |
| 575 | func (b *ExecBackend) settleClaimInstanceEnv(rt registry.Route) error { |
| 576 | p := b.pathsFor(rt) |
| 577 | adminTokenMu.Lock() |
| 578 | defer adminTokenMu.Unlock() |
| 579 | f, err := openOwnEnv(p.dataDir, rt.Host+".env", p.layout == LayoutNested, os.O_RDONLY, 0) |
| 580 | if err != nil { |
| 581 | return fmt.Errorf("open %s: %w", p.envFile, err) |
| 582 | } |
| 583 | defer f.Close() |
| 584 | st, err := f.Stat() |
| 585 | if err != nil { |
| 586 | return fmt.Errorf("stat %s: %w", p.envFile, err) |
| 587 | } |
| 588 | if st.Mode().Perm() != 0o600 { |
| 589 | return errInstanceEnvNeedsWrite |
| 590 | } |
| 591 | return syncInstanceEnv(f, p.dataDir, p.envFile) |
| 592 | } |
| 593 | |
amadan