| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/sha256" |
| 7 | "encoding/hex" |
| 8 | "encoding/json" |
| 9 | "errors" |
| 10 | "fmt" |
| 11 | "io" |
| 12 | "io/fs" |
| 13 | "os" |
| 14 | "path/filepath" |
| 15 | "reflect" |
| 16 | "sort" |
| 17 | "strconv" |
| 18 | "strings" |
| 19 | |
| 20 | "github.com/carlosframework/platform/internal/hostbox" |
| 21 | "github.com/carlosframework/platform/internal/hostclaim" |
| 22 | "github.com/carlosframework/platform/internal/hostfiles" |
| 23 | "github.com/carlosframework/platform/internal/hostwrite" |
| 24 | "github.com/carlosframework/platform/internal/instancefs" |
| 25 | "github.com/carlosframework/platform/internal/instancekey" |
| 26 | "github.com/carlosframework/platform/internal/lease" |
| 27 | "github.com/carlosframework/platform/internal/moves" |
| 28 | "github.com/carlosframework/platform/internal/regionname" |
| 29 | "github.com/carlosframework/platform/internal/release" |
| 30 | ) |
| 31 | |
| 32 | // movesFiles is the C8 files-manifest provider (claim §6.2a P6-D3, moves |
| 33 | // spec §7.6, index C8; Q3 ruling). It is two pieces of region HA's files |
| 34 | // publisher API, on hostfiles: |
| 35 | // |
| 36 | // - BuildPrepared, the prepared-manifest build an installation runs in |
| 37 | // `local` before any session exists: create-only content objects plus |
| 38 | // one create-only sequence-0 manifest under files/g-<P>/ in the |
| 39 | // location, bound to {location, P, op, stamp}, replayed by digest and |
| 40 | // key (hostfiles.Prepared). |
| 41 | // - Establish, the reader a cold admission establishes its files manifest |
| 42 | // with: the newest valid manifest of the admitted predecessor chain, |
| 43 | // read by exact version, copied create-only into the admitted |
| 44 | // generation with the env file's bytes on disk staged and the adoption |
| 45 | // item dropped (hostfiles.NewFromCopy; instance key O1'). |
| 46 | // |
| 47 | // The session-bound publisher (sequence 1 onwards under a session) stays |
| 48 | // region HA's; moves does not need it, the cut archive is the transfer. |
| 49 | // |
| 50 | // Missing or malformed data is an error, never an empty-files bootstrap |
| 51 | // (claim §6.4a). The two empty starts are the claim's own bootstrap |
| 52 | // (in.Empty, proved by EmptyAuthority before Establish runs) and a pin |
| 53 | // that evidences a host that never published files (region HA stage B's |
| 54 | // rule, hostfiles.PinnedFiles.NeverPublished, read through Evidence). |
| 55 | // Absence of a manifest, of a pin, or of a reader is never evidence: with |
| 56 | // Evidence nil, a chain without a manifest refuses every wake. |
| 57 | type movesFiles struct { |
| 58 | emptyRecovery claimEmptyRecovery |
| 59 | Seed *filesSeedOperation |
| 60 | // Files is this box's data bucket as a release.Store: current reads |
| 61 | // with the object's etag, create-only and compare-and-swap writes |
| 62 | // (the S3 store's Put with If-None-Match and If-Match). Every write |
| 63 | // this provider makes goes through it. |
| 64 | Files release.Store |
| 65 | // Data is the same bucket as a versioned store (the claim's |
| 66 | // hostbox.RequestStore in production): the chain's manifests and the |
| 67 | // prepared manifest are read at the exact version the listing or the |
| 68 | // op journal names, never by key, and BuildPrepared writes through it |
| 69 | // create-only. |
| 70 | Data lease.VersionedStore |
| 71 | // DataDir is the box's data directory, where the op journals live |
| 72 | // (moves.ReadJournal): the probe's admission reads the prepared |
| 73 | // manifest's binding from the op journal local wrote. |
| 74 | DataDir string |
| 75 | // Bucket is the data bucket's name, HomeRegion the fleet's signed home |
| 76 | // region, Region this box's engine region (park.PlacementRegion of its |
| 77 | // placement) and Box its id. Every region comparison goes through |
| 78 | // regionname.SameRegion with HomeRegion as the alias. |
| 79 | Bucket, HomeRegion, Region, Box string |
| 80 | // EnvFile resolves the host's env file on this box for the wake's |
| 81 | // database path, as the launch source that minted the instance key into |
| 82 | // it resolved it (claimEnvFileFor): the nested instance directory's for |
| 83 | // a nested row, the flat one for a flat row, and an error for anything |
| 84 | // it cannot place. |
| 85 | EnvFile func(host, dbPath string) (claimEnvFile, error) |
| 86 | // Proof proves origins against admissions the claim has recorded |
| 87 | // (platform-1e's ruling, fix round 3; claim Plan 15): in production |
| 88 | // ClaimOriginProof over the deployment store and the verified home |
| 89 | // alias, one batch call per establishment. nil refuses: there is no |
| 90 | // other route, and retained admissions are never evidence. |
| 91 | Proof movesOriginProof |
| 92 | // Evidence reads the claim pin's files evidence for a host, when the |
| 93 | // claim carries a pin: region HA's ReadPin over the op journal's pin |
| 94 | // reference. false means the claim carries no pin. nil means no pin |
| 95 | // reader is wired on this box, which reads as no evidence. |
| 96 | Evidence func(ctx context.Context, o hostclaim.Observed) (hostfiles.PinnedFiles, bool, error) |
| 97 | } |
| 98 | |
| 99 | var _ ClaimFilesManifest = (*movesFiles)(nil) |
| 100 | |
| 101 | // preparedManifest is the files_manifest journal step's value: the |
| 102 | // binding {location, P, op, stamp} of the prepared manifest an install |
| 103 | // wrote, with the digest and the exact version of its bytes. Key is |
| 104 | // <location>/files/g-<P>/manifest.json, which names the location and P. |
| 105 | // The probe's admission takes exactly Version and checks it against |
| 106 | // SHA256 (fix round 1, finding 2). |
| 107 | type preparedManifest struct { |
| 108 | Key string `json:"key"` |
| 109 | SHA256 string `json:"sha256"` |
| 110 | Version string `json:"version"` |
| 111 | Op string `json:"op"` |
| 112 | Stamp uint64 `json:"stamp"` |
| 113 | } |
| 114 | |
| 115 | // decodePreparedManifest decodes a journalled binding and holds it to the |
| 116 | // expected key, op and stamp (fix round 1, finding 3). An empty value, |
| 117 | // malformed JSON, a binding for another key, op or stamp, or one without a |
| 118 | // digest or a version, is an error: the journal is damaged or another |
| 119 | // install's, and the step is neither done nor rebuilt over it. |
| 120 | func decodePreparedManifest(raw, key, op string, stamp uint64) (preparedManifest, error) { |
| 121 | var pm preparedManifest |
| 122 | if raw == "" { |
| 123 | return pm, fmt.Errorf("%w: the journal's %s step is empty", errLandingChanged, landStepFilesManifest) |
| 124 | } |
| 125 | if err := json.Unmarshal([]byte(raw), &pm); err != nil { |
| 126 | return pm, fmt.Errorf("%w: the journal's %s step %q: %v", errLandingChanged, landStepFilesManifest, raw, err) |
| 127 | } |
| 128 | if pm.Key != key || pm.Op != op || pm.Stamp != stamp || !claimHexDigest(pm.SHA256) || pm.Version == "" || pm.Version == "null" { |
| 129 | return pm, fmt.Errorf("%w: the journal's %s step %+v is not the binding of %s by %s at stamp %d", errLandingChanged, landStepFilesManifest, pm, key, op, stamp) |
| 130 | } |
| 131 | return pm, nil |
| 132 | } |
| 133 | |
| 134 | // movesFilesPrefix is a generation's files prefix in its location, |
| 135 | // <location>/files/g-<generation>/, the rule pinFilesLiveSet and the park |
| 136 | // view's files live set share (spec §7.6), and region HA's relocation |
| 137 | // with them: moves.GenerationFilesPrefix, the one rule. |
| 138 | func movesFilesPrefix(location, generation string) string { |
| 139 | return moves.GenerationFilesPrefix(location, generation) |
| 140 | } |
| 141 | |
| 142 | // movesFilesGeneration accepts a generation name as the claim does |
| 143 | // (claimFreshGeneration's shape), never the origin or a path. |
| 144 | func movesFilesGeneration(g string) bool { |
| 145 | return g != "" && g != "." && g != ".." && g != hostclaim.RootGeneration && len(g) <= 64 && !strings.ContainsAny(g, "/\\\x00\r\n") |
| 146 | } |
| 147 | |
| 148 | // createOnlyStore is a release.Store over a versioned store for one |
| 149 | // bucket: current reads by the listing's latest version, create-only |
| 150 | // writes (PutIfAbsent), never a replace. BuildPrepared writes through it, |
| 151 | // so an install needs only the box's versioned store. |
| 152 | type createOnlyStore struct { |
| 153 | data lease.VersionedStore |
| 154 | bucket string |
| 155 | } |
| 156 | |
| 157 | func (s createOnlyStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 158 | vs, err := s.data.ListVersions(ctx, s.bucket, key) |
| 159 | if err != nil { |
| 160 | return nil, "", err |
| 161 | } |
| 162 | for _, v := range vs { |
| 163 | if v.Key == key && v.IsLatest { |
| 164 | if v.DeleteMarker || v.VersionID == "" || v.VersionID == "null" { |
| 165 | return nil, "", fmt.Errorf("host: %s is not a readable version: %w", key, os.ErrNotExist) |
| 166 | } |
| 167 | body, err := s.data.GetVersion(ctx, s.bucket, key, v.VersionID) |
| 168 | return body, v.VersionID, err |
| 169 | } |
| 170 | } |
| 171 | return nil, "", fmt.Errorf("host: %s: %w", key, os.ErrNotExist) |
| 172 | } |
| 173 | |
| 174 | func (s createOnlyStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| 175 | if ifMatch != "" { |
| 176 | return "", errors.New("host: the prepared manifest build never replaces an object") |
| 177 | } |
| 178 | id, err := s.data.PutIfAbsent(ctx, s.bucket, key, body) |
| 179 | if errors.Is(err, lease.ErrExists) { |
| 180 | return "", fmt.Errorf("%w: %s", release.ErrCASConflict, key) |
| 181 | } |
| 182 | return id, err |
| 183 | } |
| 184 | |
| 185 | func (createOnlyStore) List(context.Context, string) ([]string, error) { |
| 186 | return nil, errors.New("host: the prepared manifest build does not list") |
| 187 | } |
| 188 | func (createOnlyStore) ListDirs(context.Context, string) ([]string, error) { |
| 189 | return nil, errors.New("host: the prepared manifest build does not list") |
| 190 | } |
| 191 | func (createOnlyStore) Delete(context.Context, string) error { |
| 192 | return errors.New("host: the prepared manifest build never deletes") |
| 193 | } |
| 194 | |
| 195 | // buildPreparedManifest is BuildPrepared over a versioned store: the |
| 196 | // install's own (ir.versioned), which needs no compare-and-swap. |
| 197 | func buildPreparedManifest(ctx context.Context, data lease.VersionedStore, bucket, location, generation, op string, stamp uint64, dir, envPath string, wantEnv bool, wake hostfiles.WakeState) (preparedManifest, error) { |
| 198 | if data == nil || bucket == "" { |
| 199 | return preparedManifest{}, hostwrite.ErrUnavailable |
| 200 | } |
| 201 | return buildPrepared(ctx, createOnlyStore{data: data, bucket: bucket}, location, generation, op, stamp, dir, envPath, wantEnv, wake) |
| 202 | } |
| 203 | |
| 204 | // BuildPrepared writes the prepared manifest of generation (the |
| 205 | // installation's prepared destination generation P, g-<move id> or |
| 206 | // g-<rollback id>, never holder.generation: R19-1) under location: every |
| 207 | // regular file under dir as a create-only object at sequence 0 (a file |
| 208 | // over hostfiles.MaxFileBytes is recorded as skipped; anything not a |
| 209 | // regular file is left out, as SyncDir leaves it), the env file at envPath |
| 210 | // as the EnvItem, the wake state as the WakeItem, then the one create-only |
| 211 | // sequence-0 manifest. It returns the binding for the op journal: the key, |
| 212 | // the manifest's digest and exact version, op and stamp. A replay reuses |
| 213 | // every object by digest and reads the manifest back by key; a different |
| 214 | // manifest at the key is hostfiles.ErrManifestExists. |
| 215 | // |
| 216 | // Missing inputs are never an empty image (fix round 1, finding 4): dir |
| 217 | // must exist (the install placed it), and the env file must exist exactly |
| 218 | // when wantEnv says the install placed one (the archive's evidence, the |
| 219 | // install receipt's Env identity). A vanished tree, a vanished env file, |
| 220 | // or an env file nobody placed, refuses before any write. |
| 221 | // |
| 222 | // The caller runs it as op.exec in a phase that permits installation, |
| 223 | // after the install's own claim sync and stamp capture (local_). |
| 224 | func (f *movesFiles) BuildPrepared(ctx context.Context, location, generation, op string, stamp uint64, dir, envPath string, wantEnv bool, wake hostfiles.WakeState) (preparedManifest, error) { |
| 225 | if f == nil { |
| 226 | return preparedManifest{}, hostwrite.ErrUnavailable |
| 227 | } |
| 228 | return buildPreparedManifest(ctx, f.Data, f.Bucket, location, generation, op, stamp, dir, envPath, wantEnv, wake) |
| 229 | } |
| 230 | |
| 231 | func buildPrepared(ctx context.Context, s release.Store, location, generation, op string, stamp uint64, dir, envPath string, wantEnv bool, wake hostfiles.WakeState) (preparedManifest, error) { |
| 232 | if ctx == nil || location == "" || !movesFilesGeneration(generation) || op == "" || stamp == 0 || dir == "" || envPath == "" { |
| 233 | return preparedManifest{}, fmt.Errorf("%w: a prepared manifest names its location, generation, op, stamp, files directory and env path", hostwrite.ErrUnavailable) |
| 234 | } |
| 235 | // The inputs first, before any write: the placed tree and the env file |
| 236 | // as the install's evidence says. |
| 237 | if st, err := os.Lstat(dir); err != nil || !st.IsDir() { |
| 238 | return preparedManifest{}, fmt.Errorf("%w: the files directory %s the install placed is not there (%v)", errLandingChanged, dir, err) |
| 239 | } |
| 240 | env, err := os.ReadFile(envPath) |
| 241 | switch { |
| 242 | case err == nil && !wantEnv: |
| 243 | return preparedManifest{}, fmt.Errorf("%w: an env file is at %s, which the install did not place", errLandingChanged, envPath) |
| 244 | case errors.Is(err, fs.ErrNotExist) && wantEnv: |
| 245 | return preparedManifest{}, fmt.Errorf("%w: the env file %s the install placed is not there", errLandingChanged, envPath) |
| 246 | case err != nil && !errors.Is(err, fs.ErrNotExist): |
| 247 | return preparedManifest{}, err |
| 248 | } |
| 249 | prefix := movesFilesPrefix(location, generation) |
| 250 | p, err := hostfiles.NewPrepared(s, prefix) |
| 251 | if err != nil { |
| 252 | return preparedManifest{}, err |
| 253 | } |
| 254 | // The tree is read through an os.Root over dir, every name relative to |
| 255 | // it (#236 fix round 1): a path within the files tree's cap (4095 bytes, |
| 256 | // which the archive the install extracted holds every path to) is past |
| 257 | // PATH_MAX once the data directory is prefixed, so an absolute walk |
| 258 | // fails with ENAMETOOLONG after the cut. The root also keeps every read |
| 259 | // inside dir. |
| 260 | // |
| 261 | // Census ruling moves-overcap-cut-236 (platform-1e, 2026-10-15; |
| 262 | // binding): no read here falls back to a path-based open. A root that |
| 263 | // cannot be opened, or a name it cannot open (readFileBounded), refuses |
| 264 | // the build; nothing retries by absolute path. |
| 265 | root, err := os.OpenRoot(dir) |
| 266 | if err != nil { |
| 267 | return preparedManifest{}, fmt.Errorf("%w: the files directory %s the install placed: %v", errLandingChanged, dir, err) |
| 268 | } |
| 269 | defer root.Close() |
| 270 | err = fs.WalkDir(root.FS(), ".", func(rel string, d fs.DirEntry, err error) error { |
| 271 | if err != nil { |
| 272 | return err |
| 273 | } |
| 274 | if rel == "." || d.IsDir() || !d.Type().IsRegular() { |
| 275 | return nil |
| 276 | } |
| 277 | info, err := d.Info() |
| 278 | if err != nil { |
| 279 | return err |
| 280 | } |
| 281 | // The size before any read, and the read bounded at the limit |
| 282 | // plus one byte (fix round 1, finding 5): a file over the limit, by |
| 283 | // its stat or by growing while it is read, is recorded as skipped |
| 284 | // and never held whole in memory. |
| 285 | if info.Size() > hostfiles.MaxFileBytes { |
| 286 | return p.StageSkipped(rel, movesFilesTooLarge(info.Size())) |
| 287 | } |
| 288 | body, tooLarge, err := readFileBounded(root, rel, hostfiles.MaxFileBytes) |
| 289 | if err != nil { |
| 290 | return err |
| 291 | } |
| 292 | if tooLarge { |
| 293 | return p.StageSkipped(rel, movesFilesTooLarge(int64(len(body)))) |
| 294 | } |
| 295 | return p.Stage(ctx, rel, body, uint32(info.Mode().Perm())) |
| 296 | }) |
| 297 | if err != nil { |
| 298 | return preparedManifest{}, fmt.Errorf("prepared manifest of %s: %w", dir, err) |
| 299 | } |
| 300 | if wantEnv { |
| 301 | if err := p.StagePlatform(ctx, hostfiles.EnvItem, env); err != nil { |
| 302 | return preparedManifest{}, err |
| 303 | } |
| 304 | } |
| 305 | body, err := json.Marshal(wake) |
| 306 | if err != nil { |
| 307 | return preparedManifest{}, fmt.Errorf("marshal wake state: %w", err) |
| 308 | } |
| 309 | if err := p.StagePlatform(ctx, hostfiles.WakeItem, body); err != nil { |
| 310 | return preparedManifest{}, err |
| 311 | } |
| 312 | _, raw, version, err := p.Commit(ctx) |
| 313 | if err != nil { |
| 314 | return preparedManifest{}, err |
| 315 | } |
| 316 | if version == "" || version == "null" { |
| 317 | return preparedManifest{}, fmt.Errorf("%w: the store answered no version for %s", hostwrite.ErrUnavailable, hostfiles.ManifestKey(prefix)) |
| 318 | } |
| 319 | sum := sha256.Sum256(raw) |
| 320 | return preparedManifest{Key: hostfiles.ManifestKey(prefix), SHA256: hex.EncodeToString(sum[:]), Version: version, Op: op, Stamp: stamp}, nil |
| 321 | } |
| 322 | |
| 323 | // movesFilesTooLarge is a skipped entry's reason for a file over the |
| 324 | // limit; size is what the stat or the bounded read saw. |
| 325 | func movesFilesTooLarge(size int64) string { |
| 326 | return fmt.Sprintf("over the %d-byte limit (%d bytes or more)", hostfiles.MaxFileBytes, size) |
| 327 | } |
| 328 | |
| 329 | // readFileBounded reads rel, relative to root, whole when it holds at most |
| 330 | // limit bytes. It reads at most limit+1 bytes however large the file is or |
| 331 | // becomes while it is read; tooLarge reports that the file exceeded the |
| 332 | // limit, and the bytes returned with it are then not the whole file. A |
| 333 | // test seam. |
| 334 | var readFileBounded = func(root *os.Root, rel string, limit int64) ([]byte, bool, error) { |
| 335 | // Through the root only (census ruling moves-overcap-cut-236, binding): |
| 336 | // an open that fails is the error, never a retry by path. |
| 337 | f, err := root.Open(rel) |
| 338 | if err != nil { |
| 339 | return nil, false, err |
| 340 | } |
| 341 | defer f.Close() |
| 342 | body, err := io.ReadAll(io.LimitReader(f, limit+1)) |
| 343 | if err != nil { |
| 344 | return nil, false, err |
| 345 | } |
| 346 | return body, int64(len(body)) > limit, nil |
| 347 | } |
| 348 | |
| 349 | // movesFilesChain is the admitted predecessor chain whose manifests are in |
| 350 | // this region's bucket, newest first: the holder's generation, then each |
| 351 | // cold admission's predecessor while it is a named generation in this |
| 352 | // region (regionname.SameRegion against the home alias). A predecessor in |
| 353 | // another region ends the walk: its objects are not here, and a target's |
| 354 | // manifest never references a source-region object (P6-D3). Warm |
| 355 | // admissions keep their generation and are passed over. |
| 356 | func movesFilesChain(h *hostclaim.Holder, home, region string) []string { |
| 357 | if h == nil { |
| 358 | return nil |
| 359 | } |
| 360 | return movesFilesChainFrom(h, h.Generation, home, region) |
| 361 | } |
| 362 | |
| 363 | // recoverySourceGeneration is the generation a recovery's files baseline |
| 364 | // starts from (whole-branch fixes, item 3): for a restoring or reverting |
| 365 | // op, the source's generation the claim journalled at the flip (src_gen), |
| 366 | // which the recovery's database restore is pinned to as well; "" names a |
| 367 | // legacy source with no files generation. ok is false for any other op. |
| 368 | func recoverySourceGeneration(c *hostclaim.Claim) (string, bool) { |
| 369 | if c == nil { |
| 370 | return "", false |
| 371 | } |
| 372 | switch c.Op.Kind { |
| 373 | case hostclaim.KindRestoring, hostclaim.KindReverting: |
| 374 | return c.Op.Journal[hostclaim.JournalSrcGen], true |
| 375 | } |
| 376 | return "", false |
| 377 | } |
| 378 | |
| 379 | // restoresSourceBaseline reports whether a recovery admission restores the |
| 380 | // recorded source baseline (platform-1e's framing of item 3): a restoring |
| 381 | // op that is not a dirty return (a dirty return journals its return seal |
| 382 | // and restores the rollback generation or a certified descendant, never |
| 383 | // the stopped source), with a journalled src_gen. The files bridge from |
| 384 | // src_gen applies only to these, and only while their recovery lineage |
| 385 | // holds no manifest (the database selector's "not rooted"); every other |
| 386 | // admission follows its recovery lineage. |
| 387 | func restoresSourceBaseline(c *hostclaim.Claim) bool { |
| 388 | if c == nil || c.Op.Kind != hostclaim.KindRestoring || c.Op.Journal[moves.JournalReturnSeal] != "" { |
| 389 | return false |
| 390 | } |
| 391 | src, ok := recoverySourceGeneration(c) |
| 392 | return ok && src != "" |
| 393 | } |
| 394 | |
| 395 | // movesFilesChainFrom is movesFilesChain started at generation start |
| 396 | // instead of the holder's: start, then each cold admission's predecessor |
| 397 | // in this region, as the admissions record them. |
| 398 | func movesFilesChainFrom(h *hostclaim.Holder, start, home, region string) []string { |
| 399 | if h == nil || !movesFilesGeneration(start) { |
| 400 | return nil |
| 401 | } |
| 402 | cur := start |
| 403 | chain := []string{cur} |
| 404 | seen := map[string]bool{cur: true} |
| 405 | for i := len(h.Admissions) - 1; i >= 0; i-- { |
| 406 | a := h.Admissions[i] |
| 407 | if a.Generation != cur || a.Mode != "cold" { |
| 408 | continue |
| 409 | } |
| 410 | next := a.Predecessor |
| 411 | if !movesFilesGeneration(next) || seen[next] || !regionname.SameRegion(home, a.PredecessorRegion, region) { |
| 412 | break |
| 413 | } |
| 414 | chain = append(chain, next) |
| 415 | seen[next] = true |
| 416 | cur = next |
| 417 | } |
| 418 | return chain |
| 419 | } |
| 420 | |
| 421 | // manifestVersion is the current version of a manifest key as the listing |
| 422 | // names it: "" when the key has no current object. A delete marker or an |
| 423 | // unversioned current object is an error. |
| 424 | func (f *movesFiles) manifestVersion(ctx context.Context, key string) (string, error) { |
| 425 | return movesManifestVersion(ctx, f.Data, f.Bucket, key) |
| 426 | } |
| 427 | |
| 428 | // movesManifestVersion is manifestVersion over any versioned reader of the |
| 429 | // data bucket: the cut's preflight reads with the moves worker's own |
| 430 | // (sourceFilesRefusal), the establishment with the claim's. |
| 431 | func movesManifestVersion(ctx context.Context, data lease.VersionedStore, bucket, key string) (string, error) { |
| 432 | vs, err := data.ListVersions(ctx, bucket, key) |
| 433 | if err != nil { |
| 434 | return "", err |
| 435 | } |
| 436 | for _, v := range vs { |
| 437 | if v.Key != key || !v.IsLatest { |
| 438 | continue |
| 439 | } |
| 440 | if v.DeleteMarker || v.VersionID == "" || v.VersionID == "null" { |
| 441 | return "", fmt.Errorf("host: the files manifest %s is not a readable version", key) |
| 442 | } |
| 443 | return v.VersionID, nil |
| 444 | } |
| 445 | return "", nil |
| 446 | } |
| 447 | |
| 448 | // newestManifest is the newest valid manifest of the chain, read by exact |
| 449 | // version, with the origins its entries name (movesFilesOrigins: the |
| 450 | // location's rule, not the chain's). A generation without a manifest is |
| 451 | // passed over; the first manifest found is validated and is the baseline, |
| 452 | // malformed or not (a malformed newest manifest refuses, never an older |
| 453 | // one in its place). found is false when no generation of the chain has a |
| 454 | // manifest. |
| 455 | func (f *movesFiles) newestManifest(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, location string, chain []string) (hostfiles.Manifest, []string, bool, error) { |
| 456 | m, key, version, found, err := movesFilesSelect(ctx, f.Data, f.Bucket, location, chain) |
| 457 | if err != nil || !found { |
| 458 | return hostfiles.Manifest{}, nil, false, err |
| 459 | } |
| 460 | allowed, err := f.movesFilesAllowed(ctx, o, in, location, in.NewGeneration, m) |
| 461 | if err != nil { |
| 462 | return hostfiles.Manifest{}, nil, false, fmt.Errorf("host: the files manifest %s@%s: %w", key, version, err) |
| 463 | } |
| 464 | return m, allowed, true, nil |
| 465 | } |
| 466 | |
| 467 | // movesFilesSelect is newestManifest's selection, apart from its |
| 468 | // validation: the first generation of chain whose manifest key has a |
| 469 | // current version, read at exactly that version, with the key and the |
| 470 | // version. found is false when no generation of chain has a manifest. The |
| 471 | // establishment (newestManifest, the src_gen bridge included) and the |
| 472 | // cut's preflight (sourceFilesRefusal) both select through it, so the cut |
| 473 | // checks the manifest a recovery would restore, not a guess at it (#236). |
| 474 | func movesFilesSelect(ctx context.Context, data lease.VersionedStore, bucket, location string, chain []string) (m hostfiles.Manifest, key, version string, found bool, err error) { |
| 475 | for _, g := range chain { |
| 476 | prefix := movesFilesPrefix(location, g) |
| 477 | key := hostfiles.ManifestKey(prefix) |
| 478 | version, err := movesManifestVersion(ctx, data, bucket, key) |
| 479 | if err != nil { |
| 480 | return hostfiles.Manifest{}, "", "", false, err |
| 481 | } |
| 482 | if version == "" { |
| 483 | continue |
| 484 | } |
| 485 | m, found, err := hostfiles.Load(ctx, versionStore{data: data, bucket: bucket, key: key, version: version}, prefix) |
| 486 | if err != nil { |
| 487 | return hostfiles.Manifest{}, "", "", false, err |
| 488 | } |
| 489 | if !found { |
| 490 | return hostfiles.Manifest{}, "", "", false, fmt.Errorf("host: the files manifest %s@%s is not readable", key, version) |
| 491 | } |
| 492 | return m, key, version, true, nil |
| 493 | } |
| 494 | return hostfiles.Manifest{}, "", "", false, nil |
| 495 | } |
| 496 | |
| 497 | // movesFilesRestorable is the reader's path cap as moves applies it |
| 498 | // (hostfiles.RestorableWithinCap, the rule checkManifest applies to every |
| 499 | // entry, live, tombstoned or skipped): a manifest that fails it is |
| 500 | // hostfiles.ErrTooDeep, and no establishment restores it. The recovery's |
| 501 | // establishment (movesFilesAllowed) and the cut's preflight |
| 502 | // (sourceFilesRefusal) both call it, so a move whose recovery could not |
| 503 | // restore the source's files never starts its cut (#236; Paul's ruling of |
| 504 | // 2026-10-15). If region HA relaxes the readers' rule, RestorableWithinCap |
| 505 | // follows it, and both sides here with it. |
| 506 | func movesFilesRestorable(m hostfiles.Manifest) error { |
| 507 | return hostfiles.RestorableWithinCap(m) |
| 508 | } |
| 509 | |
| 510 | // movesOriginProof proves that generations of the holder's incarnation |
| 511 | // were admitted, by the claim's own records (platform-1e's ruling, fix |
| 512 | // round 3; claim Plan 15): provenance comes from the claim, never from a |
| 513 | // manifest, whose writer holds the bucket credentials that could write |
| 514 | // its Origins too. The implementation is Plan 15's ClaimOriginProof: a |
| 515 | // batch Proven over the whole wanted set, which authenticates the bound |
| 516 | // observation against the stored claim (the CAS result the caller holds, |
| 517 | // never a bare Admission.NewGeneration), walks the claim's recorded |
| 518 | // admissions once from its committed head, stops as soon as every wanted |
| 519 | // generation is proven, bounds each node read, and takes home aliases only |
| 520 | // through the verified roster binding. The chain starts empty: nothing is |
| 521 | // seeded from retained admissions, which are never evidence, and a claim |
| 522 | // with no recorded head proves nothing, so every existing host refuses |
| 523 | // until the #250 files backfill has run. That is intended. A false answer |
| 524 | // is ErrOrigin; an error is the error. |
| 525 | type movesOriginProof interface { |
| 526 | Proven(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, generations []string) (map[string]bool, error) |
| 527 | } |
| 528 | |
| 529 | // proveOrigins is the one route to origin proof (fix round 4; Plan 15): |
| 530 | // one Proven call over the deduplicated set of generations, and a |
| 531 | // generation's files prefix under location enters the allowed set only |
| 532 | // when that call proves it. A generation the proof does not prove is |
| 533 | // ErrOrigin, naming the first; an error is the error; no proof wired |
| 534 | // refuses. No per-origin calls (Plan 15's review: one walk per |
| 535 | // establishment). |
| 536 | func (f *movesFiles) proveOrigins(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, location string, generations []string) ([]string, error) { |
| 537 | if f.Proof == nil { |
| 538 | return nil, fmt.Errorf("%w: no origin proof is wired", hostwrite.ErrUnavailable) |
| 539 | } |
| 540 | set := make([]string, 0, len(generations)) |
| 541 | seen := map[string]bool{} |
| 542 | for _, gen := range generations { |
| 543 | if !movesFilesGeneration(gen) { |
| 544 | return nil, fmt.Errorf("%w: %q is not a generation name", hostfiles.ErrOrigin, gen) |
| 545 | } |
| 546 | if !seen[gen] { |
| 547 | seen[gen] = true |
| 548 | set = append(set, gen) |
| 549 | } |
| 550 | } |
| 551 | proven, err := f.Proof.Proven(ctx, o, in, set) |
| 552 | if err != nil { |
| 553 | return nil, fmt.Errorf("prove the origins of %s: %w", location, err) |
| 554 | } |
| 555 | origins := make([]string, 0, len(set)) |
| 556 | for _, gen := range set { |
| 557 | if !proven[gen] { |
| 558 | return nil, fmt.Errorf("%w: %s is not proven by the claim's recorded admissions", hostfiles.ErrOrigin, movesFilesPrefix(location, gen)) |
| 559 | } |
| 560 | origins = append(origins, movesFilesPrefix(location, gen)) |
| 561 | } |
| 562 | // The prefixes are sorted, not the generation names: the two do not |
| 563 | // order alike ("mv1" before "mv1-old", "g-mv1-old/" before "g-mv1/"), |
| 564 | // and this is the carried set SetOrigins stores and establishedOver |
| 565 | // compares (Astra's finding on the Plan 15 switch). |
| 566 | sort.Strings(origins) |
| 567 | return origins, nil |
| 568 | } |
| 569 | |
| 570 | // movesFilesAllowed is the origin set an admission allows for m, the |
| 571 | // manifest it establishes from (Task 6b, the moves side of claim Plan 15's |
| 572 | // review): the origins m's entries actually reference, each proven against |
| 573 | // the claim's records through proveOrigin after the location rule |
| 574 | // (movesFilesOrigins), plus the one generation this admission opens |
| 575 | // (admitted: proven through the same route, round 4's binding, one call). |
| 576 | // Nothing else is proven and nothing else is carried: the retained chain's |
| 577 | // other generations are not candidates, and a carried hint |
| 578 | // (Manifest.Origins) that no entry references is neither proven nor |
| 579 | // carried forward, so the proof's cost follows the live references, not |
| 580 | // the history. A referenced origin the proof cannot prove is ErrOrigin. |
| 581 | // Then ValidateManifest over the proven set, which is the new manifest's |
| 582 | // carried set (SetOrigins). Nothing refused before is accepted now: every |
| 583 | // referenced origin and the admitted generation go through the proof as |
| 584 | // before; only unreferenced hints and unreferenced chain generations, which |
| 585 | // no entry depends on, are no longer asked about. |
| 586 | func (f *movesFiles) movesFilesAllowed(ctx context.Context, o hostclaim.Observed, in hostclaim.Admission, location, admitted string, m hostfiles.Manifest) ([]string, error) { |
| 587 | // The path cap first, by the rule the cut's preflight applies |
| 588 | // (movesFilesRestorable): ValidateManifest below refuses the same |
| 589 | // manifest with the same error, so this only refuses it before any |
| 590 | // origin proof, and keeps the two sides on one function. |
| 591 | if err := movesFilesRestorable(m); err != nil { |
| 592 | return nil, err |
| 593 | } |
| 594 | named, err := movesFilesOrigins(location, "", m) |
| 595 | if err != nil { |
| 596 | return nil, err |
| 597 | } |
| 598 | base := moves.FilesGenerationBase(location) |
| 599 | generations := []string{admitted} |
| 600 | for _, origin := range named { |
| 601 | rest, under := strings.CutPrefix(origin, base) |
| 602 | gen, slash := strings.CutSuffix(rest, "/") |
| 603 | if !under || !slash || !movesFilesGeneration(gen) { |
| 604 | return nil, fmt.Errorf("%w: %s is not a generation's files prefix of %s", hostfiles.ErrOrigin, origin, location) |
| 605 | } |
| 606 | generations = append(generations, gen) |
| 607 | } |
| 608 | // One Proven call over the deduplicated referenced origins plus the |
| 609 | // admitted generation (Plan 15). |
| 610 | allowed, err := f.proveOrigins(ctx, o, in, location, generations) |
| 611 | if err != nil { |
| 612 | return nil, err |
| 613 | } |
| 614 | if err := hostfiles.ValidateManifest(m, allowed); err != nil { |
| 615 | return nil, err |
| 616 | } |
| 617 | return allowed, nil |
| 618 | } |
| 619 | |
| 620 | // movesFilesOrigins is the set of origins a manifest's objects name, each |
| 621 | // held to the claim's provenance rule (P6-D3): an immutable object within |
| 622 | // the authorized incarnation location in this region's bucket, under a |
| 623 | // generation's files prefix, <location>/files/g-<generation>/. A manifest |
| 624 | // naming nothing has its own prefix as its one origin. The admitted chain |
| 625 | // selects which manifest is newest; it does not bound the origins that |
| 626 | // manifest may name, because AdmitWriter keeps eight admissions while a |
| 627 | // copied manifest carries its oldest objects for as long as they are live |
| 628 | // (fix round 1, finding 6). The same location rule is the files live |
| 629 | // set's (pinFilesLiveSet), so every object a live manifest names stays |
| 630 | // pinned under it. |
| 631 | func movesFilesOrigins(location, prefix string, m hostfiles.Manifest) ([]string, error) { |
| 632 | base := moves.FilesGenerationBase(location) |
| 633 | seen := map[string]bool{} |
| 634 | var out []string |
| 635 | for _, entries := range []map[string]hostfiles.Entry{m.Entries, m.Platform} { |
| 636 | for name, e := range entries { |
| 637 | if e.Object == "" || e.Deleted || e.Skipped != "" { |
| 638 | continue |
| 639 | } |
| 640 | origin, _, ok := strings.Cut(e.Object, "objects/") |
| 641 | rest, under := strings.CutPrefix(origin, base) |
| 642 | gen, slash := strings.CutSuffix(rest, "/") |
| 643 | if !ok || !under || !slash || !movesFilesGeneration(gen) { |
| 644 | return nil, fmt.Errorf("%w: %s names %s, not an object under a generation's files prefix of %s", hostfiles.ErrOrigin, name, e.Object, location) |
| 645 | } |
| 646 | if !seen[origin] { |
| 647 | seen[origin] = true |
| 648 | out = append(out, origin) |
| 649 | } |
| 650 | } |
| 651 | } |
| 652 | if len(out) == 0 && prefix != "" { |
| 653 | out = []string{prefix} |
| 654 | } |
| 655 | return out, nil |
| 656 | } |
| 657 | |
| 658 | // openPrepared is the probe's baseline (in.Prepared, or a relocating |
| 659 | // op's opening of P, preparedOpening): P's prepared manifest at exactly |
| 660 | // the version its installation's binding names (fix round 1, finding 2). |
| 661 | // The binding is read by preparedBinding, which chooses its reader by the |
| 662 | // claim's op kind, and is held to P's key, the claim's op and the |
| 663 | // installation's stamp; the bytes are read at that version, |
| 664 | // checked against the bound digest, and validated as a prepared |
| 665 | // manifest (sequence 0, objects within the location by the claim's rule, |
| 666 | // movesFilesOrigins: a cross-region landing's names destination-local |
| 667 | // objects by its build, a same-region relocation's may retain source |
| 668 | // generation objects). The key must |
| 669 | // currently hold either those bytes (the first establishment, which the |
| 670 | // returned publisher replaces by compare-and-swap) or this box's own |
| 671 | // establishment of them (a retry after a crash past the commit: the same |
| 672 | // entries, the env bytes on disk, no adoption item, which the publisher |
| 673 | // then leaves as it is). Anything else refuses before any write. |
| 674 | func (f *movesFiles) openPrepared(ctx context.Context, files release.Store, o hostclaim.Observed, in hostclaim.Admission, gen, prefix string, env []byte) (*hostfiles.Publisher, error) { |
| 675 | if f.DataDir == "" { |
| 676 | return nil, hostwrite.ErrUnavailable |
| 677 | } |
| 678 | h := o.Claim.Holder |
| 679 | key := hostfiles.ManifestKey(prefix) |
| 680 | pm, err := f.preparedBinding(o, gen, key) |
| 681 | if err != nil { |
| 682 | return nil, err |
| 683 | } |
| 684 | exact, err := f.Data.GetVersion(ctx, f.Bucket, key, pm.Version) |
| 685 | if err != nil { |
| 686 | return nil, fmt.Errorf("host: the prepared files manifest %s@%s: %w", key, pm.Version, err) |
| 687 | } |
| 688 | if sum := sha256.Sum256(exact); hex.EncodeToString(sum[:]) != pm.SHA256 { |
| 689 | return nil, fmt.Errorf("%w: the prepared files manifest %s@%s is not the journalled bytes", hostbox.ErrChanged, key, pm.Version) |
| 690 | } |
| 691 | prepared, found, err := hostfiles.Load(ctx, exactBytesStore{key: key, body: exact, version: pm.Version}, prefix) |
| 692 | if err != nil || !found { |
| 693 | return nil, fmt.Errorf("host: the prepared files manifest %s@%s: %v", key, pm.Version, err) |
| 694 | } |
| 695 | // The prepared manifest's allowed set: the origins its entries |
| 696 | // reference, proven, plus P itself (its carried set, seeded by the |
| 697 | // build with its own prefix, is a hint only; an old one carries none). |
| 698 | if len(prepared.Origins) == 0 { |
| 699 | prepared.Origins = []string{prefix} |
| 700 | } |
| 701 | allowed, err := f.movesFilesAllowed(ctx, o, in, h.Location, gen, prepared) |
| 702 | if err != nil { |
| 703 | return nil, fmt.Errorf("host: the prepared files manifest %s@%s: %w", key, pm.Version, err) |
| 704 | } |
| 705 | if prepared.Seq != 0 { |
| 706 | return nil, fmt.Errorf("%w: the prepared files manifest %s@%s is at sequence %d, not 0", hostbox.ErrChanged, key, pm.Version, prepared.Seq) |
| 707 | } |
| 708 | // One read of the key: the bytes and the token the publisher is built |
| 709 | // from are the ones checked here, never a second read (fix round 2, |
| 710 | // finding 2). A replacement after this read fails the commit's |
| 711 | // compare-and-swap. |
| 712 | cur, token, err := files.Get(ctx, key) |
| 713 | if err != nil { |
| 714 | return nil, err |
| 715 | } |
| 716 | if !bytes.Equal(cur, exact) { |
| 717 | current, found, err := hostfiles.Load(ctx, exactBytesStore{key: key, body: cur, version: token}, prefix) |
| 718 | if err != nil || !found { |
| 719 | return nil, fmt.Errorf("host: the files manifest %s: %v", key, err) |
| 720 | } |
| 721 | if err := establishedOver(ctx, files, h.Location, prefix, allowed, current, prepared, env); err != nil { |
| 722 | return nil, fmt.Errorf("%w: the files manifest %s is neither the prepared manifest %s nor this box's establishment of it: %v", hostbox.ErrChanged, key, pm.Version, err) |
| 723 | } |
| 724 | } |
| 725 | p, err := hostfiles.OpenAt(files, prefix, cur, token) |
| 726 | if err != nil { |
| 727 | return nil, err |
| 728 | } |
| 729 | p.SetOrigins(allowed) |
| 730 | return p, nil |
| 731 | } |
| 732 | |
| 733 | // preparedBinding is the binding of P's prepared manifest, read by the |
| 734 | // reader the claim's op kind names, never by which file happens to exist: |
| 735 | // a move's install journals it in the moves op journal |
| 736 | // (.moves/<host>/<P>/journal.json, the files_manifest step); a |
| 737 | // relocation's install records it in its own install journal |
| 738 | // (relocate/ops/<op>/install.json), which the claim's local_install |
| 739 | // receipt binds by digest, and the claim's prepared_authority binds its |
| 740 | // bytes. Either way the binding names key, op, stamp, digest and version. |
| 741 | // The absence of one reader's file is never permission to use the other's, |
| 742 | // and any other kind opens no prepared generation. |
| 743 | func (f *movesFiles) preparedBinding(o hostclaim.Observed, gen, key string) (preparedManifest, error) { |
| 744 | switch kind := o.Claim.Op.Kind; { |
| 745 | case kind == hostclaim.KindRelocating: |
| 746 | return relocationPreparedManifest(f.DataDir, f.Box, o, gen, key) |
| 747 | case claimMovesKind(kind): |
| 748 | return movesPreparedManifest(f.DataDir, o, gen, key) |
| 749 | default: |
| 750 | return preparedManifest{}, fmt.Errorf("%w: a %s claim opens no prepared generation", hostbox.ErrChanged, kind) |
| 751 | } |
| 752 | } |
| 753 | |
| 754 | // movesPreparedManifest is a move's binding: the op journal's |
| 755 | // files_manifest step, held to P's key, the claim's op and the |
| 756 | // installation's stamp (installationStamp). |
| 757 | func movesPreparedManifest(dataDir string, o hostclaim.Observed, gen, key string) (preparedManifest, error) { |
| 758 | j, err := moves.ReadJournal(dataDir, o.Host, gen) |
| 759 | if err != nil { |
| 760 | return preparedManifest{}, err |
| 761 | } |
| 762 | raw, ok := j.Steps[landStepFilesManifest] |
| 763 | if !ok { |
| 764 | return preparedManifest{}, fmt.Errorf("%w: the op journal of %s/%s has no prepared files manifest", errLandingChanged, o.Host, gen) |
| 765 | } |
| 766 | stamp, err := installationStamp(o.Claim) |
| 767 | if err != nil { |
| 768 | return preparedManifest{}, err |
| 769 | } |
| 770 | return decodePreparedManifest(raw, key, o.Claim.Op.ID, stamp) |
| 771 | } |
| 772 | |
| 773 | // relocationPreparedManifest is a relocation's binding (region HA §5.8). |
| 774 | // It holds two stamps apart, each exactly. P's build stamp is the one the |
| 775 | // claim's prepared_authority binds (install_stamp when ToInstalled ran): |
| 776 | // the authority names P, that stamp and the digest of P's prepared files |
| 777 | // manifest (sequence 0), and the install journal's files_manifest step |
| 778 | // binds P's key, the op, that build stamp, the digest and the exact |
| 779 | // version. This box's install stamp is the claim's current install_stamp, |
| 780 | // which a replacement target issues anew while keeping P and its |
| 781 | // authority (hostclaim ReplaceTarget at installed, or in probing before |
| 782 | // P's first admission): the holder's stamp (the probe's admission keeps |
| 783 | // it), the claim's local_install receipt (this box, that stamp, the |
| 784 | // digest of its install journal) and the journal itself (read whole, held |
| 785 | // to the receipt's digest, naming the host, the op, this box, that stamp |
| 786 | // and P) all carry it. The build precedes every install, so the build |
| 787 | // stamp is never past the install stamp. The step's digest must be the |
| 788 | // authority's. Anything missing or different refuses; neither stamp |
| 789 | // stands in for the other. |
| 790 | func relocationPreparedManifest(dataDir, box string, o hostclaim.Observed, gen, key string) (preparedManifest, error) { |
| 791 | c := o.Claim |
| 792 | auth, err := preparedAuthority(c) |
| 793 | if err != nil { |
| 794 | return preparedManifest{}, err |
| 795 | } |
| 796 | stamp, err := relocationInstallStamp(c) |
| 797 | if err != nil { |
| 798 | return preparedManifest{}, err |
| 799 | } |
| 800 | if auth.Generation != gen || auth.FilesSeq != 0 || !claimHexDigest(auth.FilesManifestSHA256) || auth.InstallStamp == 0 || auth.InstallStamp > stamp || c.Holder.Stamp != stamp { |
| 801 | return preparedManifest{}, fmt.Errorf("%w: the prepared authority %+v is not a build of P %s at or before the install stamp %d", hostbox.ErrChanged, auth, gen, stamp) |
| 802 | } |
| 803 | var receipt hostclaim.LocalInstall |
| 804 | if json.Unmarshal([]byte(c.Op.Journal["local_install"]), &receipt) != nil || receipt.Box != box || receipt.InstallStamp != stamp || !claimHexDigest(receipt.JournalSHA256) { |
| 805 | return preparedManifest{}, fmt.Errorf("%w: the claim journals no local install receipt of this box at the install stamp", hostbox.ErrChanged) |
| 806 | } |
| 807 | path, err := relocationInstallPath(dataDir, c.Op.ID) |
| 808 | if err != nil { |
| 809 | return preparedManifest{}, err |
| 810 | } |
| 811 | raw, err := os.ReadFile(path) |
| 812 | if err != nil { |
| 813 | return preparedManifest{}, fmt.Errorf("host: the relocation's install journal: %w", err) |
| 814 | } |
| 815 | if hostfilesSum(raw) != receipt.JournalSHA256 { |
| 816 | return preparedManifest{}, fmt.Errorf("%w: %s is not the install journal the claim's receipt names", hostbox.ErrChanged, path) |
| 817 | } |
| 818 | var j relocationInstallJournal |
| 819 | if err := json.Unmarshal(raw, &j); err != nil { |
| 820 | return preparedManifest{}, fmt.Errorf("%w: %s: %v", hostbox.ErrChanged, path, err) |
| 821 | } |
| 822 | if j.V != 1 || j.Host != o.Host || j.Op != c.Op.ID || j.Box != box || j.InstallStamp != stamp || j.Prepared != gen { |
| 823 | return preparedManifest{}, fmt.Errorf("%w: %s is not this box's install of %s at the install stamp", hostbox.ErrChanged, path, gen) |
| 824 | } |
| 825 | step, ok := j.Steps[relocationStepFilesManifest] |
| 826 | if !ok { |
| 827 | return preparedManifest{}, fmt.Errorf("%w: the install journal of %s has no prepared files manifest", errLandingChanged, c.Op.ID) |
| 828 | } |
| 829 | pm, err := decodePreparedManifest(step, key, c.Op.ID, auth.InstallStamp) |
| 830 | if err != nil { |
| 831 | return preparedManifest{}, err |
| 832 | } |
| 833 | if pm.SHA256 != auth.FilesManifestSHA256 { |
| 834 | return preparedManifest{}, fmt.Errorf("%w: the install bound %s at %s, not the prepared authority's digest", hostbox.ErrChanged, key, pm.SHA256) |
| 835 | } |
| 836 | return pm, nil |
| 837 | } |
| 838 | |
| 839 | // preparedOpening reports whether this admission opens a prepared |
| 840 | // generation whose files manifest an installation built. For a move it is |
| 841 | // in.Prepared. For a relocation it is the probe's admission that opens the |
| 842 | // journalled P, whether or not the database is empty: relocation always |
| 843 | // builds P's files manifest (§5.8), so database emptiness (in.Empty) and |
| 844 | // files preparation are independent, and an empty database still starts |
| 845 | // from P's files. A relocating admission that says prepared or empty for |
| 846 | // any other generation is refused. |
| 847 | func preparedOpening(o hostclaim.Observed, in ClaimLaunchInputs) (bool, error) { |
| 848 | c := o.Claim |
| 849 | if c.Op.Kind != hostclaim.KindRelocating { |
| 850 | return in.Prepared, nil |
| 851 | } |
| 852 | p := c.Op.Journal["prepared"] |
| 853 | opens := p != "" && c.Op.Phase == hostclaim.PhaseRelocatingProbing && in.Admission.NewGeneration == p |
| 854 | if (in.Prepared || in.Empty) && !opens { |
| 855 | return false, fmt.Errorf("%w: a relocating admission is prepared or empty only when it opens P", hostbox.ErrChanged) |
| 856 | } |
| 857 | return opens, nil |
| 858 | } |
| 859 | |
| 860 | // installationStamp is the stamp the installation that wrote P's prepared |
| 861 | // manifest captured (ir.capture at local), validated apart from the |
| 862 | // admitted session's authority (whole-branch fixes, item 2). A return's |
| 863 | // install (restoring, or a revert's) ran at the source installation's |
| 864 | // stamp S, which the claim journals at the flip (source_stamp) and the |
| 865 | // recovery's cold admission then replaces with a new one; a landing's |
| 866 | // install ran at the flip's stamp, which the first probe's admission |
| 867 | // keeps (KeepStamp, since no writer was admitted between the installing |
| 868 | // CAS and the probe), so it is the holder's. |
| 869 | func installationStamp(c *hostclaim.Claim) (uint64, error) { |
| 870 | switch c.Op.Kind { |
| 871 | case hostclaim.KindRestoring, hostclaim.KindReverting: |
| 872 | stamp, err := strconv.ParseUint(c.Op.Journal[hostclaim.JournalSourceStamp], 10, 64) |
| 873 | if err != nil || stamp == 0 { |
| 874 | return 0, fmt.Errorf("%w: the claim journals no source installation stamp for %s", errLandingChanged, c.Op.ID) |
| 875 | } |
| 876 | return stamp, nil |
| 877 | } |
| 878 | if c.Holder == nil || c.Holder.Stamp == 0 { |
| 879 | return 0, fmt.Errorf("%w: the holder has no installation stamp", errLandingChanged) |
| 880 | } |
| 881 | return c.Holder.Stamp, nil |
| 882 | } |
| 883 | |
| 884 | // establishedOver reports whether cur is this box's establishment of |
| 885 | // prepared, in full (fix round 2 finding 6, fix round 3): exactly what |
| 886 | // this establishment would have written, derived from the prepared |
| 887 | // baseline and the establishment's own operations by the publisher's |
| 888 | // rules, never from the replay's own numbers. From prepared's sequence, |
| 889 | // the env file's bytes take a new object at the next sequence when they |
| 890 | // differ from prepared's env item; the adoption item's deletion takes the |
| 891 | // sequence after when prepared carried one; the manifest's sequence is |
| 892 | // the last handed out; the entries and other platform items are |
| 893 | // prepared's; the carried set is the admitted one. cur must equal that |
| 894 | // manifest field for field, and the env object must be at its key with |
| 895 | // the env's digest. A replay whose env entry names an object elsewhere, a |
| 896 | // different object under P, or a fabricated sequence whose object does |
| 897 | // not exist, is not this box's establishment however its digest reads. |
| 898 | func establishedOver(ctx context.Context, files release.Store, location, prefix string, allowed []string, cur, prepared hostfiles.Manifest, env []byte) error { |
| 899 | want := hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}, Origins: allowed} |
| 900 | for k, v := range prepared.Entries { |
| 901 | want.Entries[k] = v |
| 902 | } |
| 903 | for k, v := range prepared.Platform { |
| 904 | want.Platform[k] = v |
| 905 | } |
| 906 | next := prepared.Seq |
| 907 | envSHA := hostfilesSum(env) |
| 908 | if e, ok := prepared.Platform[hostfiles.EnvItem]; !ok || e.SHA256 != envSHA { |
| 909 | next++ |
| 910 | want.Platform[hostfiles.EnvItem] = hostfiles.Entry{Object: fmt.Sprintf("%sobjects/%016d-%s", prefix, next, envSHA), SHA256: envSHA, Size: int64(len(env)), Mode: 0o600, Seq: next} |
| 911 | } |
| 912 | if _, has := prepared.Platform[instancekey.AdoptionItem]; has { |
| 913 | next++ |
| 914 | delete(want.Platform, instancekey.AdoptionItem) |
| 915 | } |
| 916 | want.Seq = next |
| 917 | if cur.Entries == nil { |
| 918 | cur.Entries = map[string]hostfiles.Entry{} |
| 919 | } |
| 920 | if cur.Platform == nil { |
| 921 | cur.Platform = map[string]hostfiles.Entry{} |
| 922 | } |
| 923 | if !reflect.DeepEqual(cur, want) { |
| 924 | return fmt.Errorf("the manifest %+v is not what this establishment writes over the prepared one (%+v)", cur, want) |
| 925 | } |
| 926 | if _, err := movesFilesOrigins(location, prefix, cur); err != nil { |
| 927 | return err |
| 928 | } |
| 929 | if err := hostfiles.ValidateManifest(cur, allowed); err != nil { |
| 930 | return err |
| 931 | } |
| 932 | e := want.Platform[hostfiles.EnvItem] |
| 933 | body, _, err := files.Get(ctx, e.Object) |
| 934 | if err != nil { |
| 935 | return fmt.Errorf("the env object %s: %w", e.Object, err) |
| 936 | } |
| 937 | if hostfilesSum(body) != e.SHA256 || int64(len(body)) != e.Size { |
| 938 | return fmt.Errorf("the env object %s is not the env file's bytes", e.Object) |
| 939 | } |
| 940 | return nil |
| 941 | } |
| 942 | |
| 943 | func hostfilesSum(body []byte) string { |
| 944 | sum := sha256.Sum256(body) |
| 945 | return hex.EncodeToString(sum[:]) |
| 946 | } |
| 947 | |
| 948 | // exactBytesStore is a read-only release.Store holding one object's exact |
| 949 | // bytes at a version already read: hostfiles.Load's reader for a manifest |
| 950 | // the caller verified. |
| 951 | type exactBytesStore struct { |
| 952 | key, version string |
| 953 | body []byte |
| 954 | } |
| 955 | |
| 956 | func (s exactBytesStore) Get(_ context.Context, key string) ([]byte, string, error) { |
| 957 | if key != s.key { |
| 958 | return nil, "", os.ErrNotExist |
| 959 | } |
| 960 | return s.body, s.version, nil |
| 961 | } |
| 962 | func (exactBytesStore) Put(context.Context, string, []byte, string) (string, error) { |
| 963 | return "", errors.New("host: the prepared manifest's exact bytes are read only") |
| 964 | } |
| 965 | func (exactBytesStore) List(context.Context, string) ([]string, error) { |
| 966 | return nil, errors.New("host: the prepared manifest's exact bytes are read only") |
| 967 | } |
| 968 | func (exactBytesStore) ListDirs(context.Context, string) ([]string, error) { |
| 969 | return nil, errors.New("host: the prepared manifest's exact bytes are read only") |
| 970 | } |
| 971 | func (exactBytesStore) Delete(context.Context, string) error { |
| 972 | return errors.New("host: the prepared manifest's exact bytes are read only") |
| 973 | } |
| 974 | |
| 975 | // selectionStore is the data bucket as Establish writes it (fix round 1, |
| 976 | // finding 7): every write is fenced by the start's captured selection |
| 977 | // immediately before the request (work.With with no effect, the |
| 978 | // writerFence idiom: the lock is taken for the check and released before |
| 979 | // the request), and every request is bounded by hostbox.RequestLimit. No |
| 980 | // request is made under the lock, so a stalled request never delays |
| 981 | // BeginFence; a selection that no longer holds refuses the write. |
| 982 | type selectionStore struct { |
| 983 | release.Store |
| 984 | fence func(context.Context) error |
| 985 | } |
| 986 | |
| 987 | func (s selectionStore) bound(ctx context.Context) (context.Context, context.CancelFunc) { |
| 988 | return context.WithTimeout(ctx, hostbox.RequestLimit) |
| 989 | } |
| 990 | |
| 991 | func (s selectionStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 992 | ctx, cancel := s.bound(ctx) |
| 993 | defer cancel() |
| 994 | return s.Store.Get(ctx, key) |
| 995 | } |
| 996 | |
| 997 | func (s selectionStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| 998 | if err := s.fence(ctx); err != nil { |
| 999 | return "", err |
| 1000 | } |
| 1001 | ctx, cancel := s.bound(ctx) |
| 1002 | defer cancel() |
| 1003 | return s.Store.Put(ctx, key, body, ifMatch) |
| 1004 | } |
| 1005 | |
| 1006 | // emptyManifest is the base of an empty start. |
| 1007 | func emptyManifest() hostfiles.Manifest { |
| 1008 | return hostfiles.Manifest{V: hostfiles.ManifestVersion, Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}} |
| 1009 | } |
| 1010 | |
| 1011 | // Establish is ClaimFilesManifest.Establish. It runs once per start under |
| 1012 | // the start's captured selection, after the launch source has minted the |
| 1013 | // instance key into <host>.env and before the app executes. The env file |
| 1014 | // is the one the launch source minted into, found through the host's row |
| 1015 | // (EnvFile; a nested row's is in its instance directory). |
| 1016 | // |
| 1017 | // The baseline: |
| 1018 | // - a prepared opening (preparedOpening: in.Prepared for a move, the |
| 1019 | // probe's admission opening P for a relocation, whatever its |
| 1020 | // database): P's own prepared manifest at the version its |
| 1021 | // installation bound (preparedBinding, by the op's kind), read current |
| 1022 | // with its etag so the env revision replaces exactly it (a new |
| 1023 | // version; the sequence-0 version stays), validated against the |
| 1024 | // origins its entries name; |
| 1025 | // - in.Empty (the claim's empty bootstrap, EmptyAuthority already run): |
| 1026 | // no manifest; |
| 1027 | // - otherwise the newest valid manifest of the chain, read by exact |
| 1028 | // version; with none, the pin's never-published evidence or an error |
| 1029 | // (hostfiles.ErrNoEvidence), never an empty bootstrap. |
| 1030 | // |
| 1031 | // Then the env file's exact bytes on disk are staged as the EnvItem, the |
| 1032 | // adoption item is dropped and the revision is committed (create-only for |
| 1033 | // a new generation; compare-and-swap on P), each bucket write fenced by |
| 1034 | // the start's selection immediately before it and bounded by the request |
| 1035 | // limit, never under the lock the work holds (selectionStore; fix round |
| 1036 | // 1, finding 7). Then, under work, the local adoption record is removed: |
| 1037 | // the one local effect. Establish never writes the env file. An error, a |
| 1038 | // bucket that cannot be reached included, fails the wake. |
| 1039 | func (f *movesFiles) Establish(ctx context.Context, o hostclaim.Observed, in ClaimLaunchInputs, work hostwrite.Work) error { |
| 1040 | if f == nil || f.Files == nil || f.Data == nil || f.Bucket == "" || f.Region == "" || f.Box == "" || f.EnvFile == nil || ctx == nil || work == nil { |
| 1041 | return hostwrite.ErrUnavailable |
| 1042 | } |
| 1043 | if o.Absent || o.Claim == nil || o.Claim.Holder == nil || o.Host == "" || in.Identity.Config == "" || in.Identity.Release == "" { |
| 1044 | return hostbox.ErrChanged |
| 1045 | } |
| 1046 | h := o.Claim.Holder |
| 1047 | gen := in.Admission.NewGeneration |
| 1048 | if in.Admission.Box != f.Box || !regionname.SameRegion(f.HomeRegion, in.Admission.Region, f.Region) || !movesFilesGeneration(gen) || h.Location == "" { |
| 1049 | return hostbox.ErrChanged |
| 1050 | } |
| 1051 | envFile, err := f.EnvFile(o.Host, in.DBPath) |
| 1052 | if err != nil { |
| 1053 | return err |
| 1054 | } |
| 1055 | env, err := readClaimEnvFile(envFile) |
| 1056 | if err != nil { |
| 1057 | return fmt.Errorf("host: the env file the files manifest must carry: %w", err) |
| 1058 | } |
| 1059 | prepared, err := preparedOpening(o, in) |
| 1060 | if err != nil { |
| 1061 | return err |
| 1062 | } |
| 1063 | |
| 1064 | prefix := movesFilesPrefix(h.Location, gen) |
| 1065 | var storage release.Store = f.Files |
| 1066 | if f.Seed != nil { |
| 1067 | if err := f.Seed.check(ctx, o, in); err != nil { |
| 1068 | return err |
| 1069 | } |
| 1070 | storage = filesSeedCheckingStore{Store: f.Files, data: f.Data, bucket: f.Bucket, prefix: prefix} |
| 1071 | } |
| 1072 | files := selectionStore{Store: storage, fence: writerFence(work, in.Identity)} |
| 1073 | var p *hostfiles.Publisher |
| 1074 | switch { |
| 1075 | case prepared: |
| 1076 | p, err = f.openPrepared(ctx, files, o, in.Admission, gen, prefix, env) |
| 1077 | if err != nil { |
| 1078 | return err |
| 1079 | } |
| 1080 | case in.Empty: |
| 1081 | own, err := f.proveOrigins(ctx, o, in.Admission, h.Location, []string{gen}) |
| 1082 | if err != nil { |
| 1083 | return err |
| 1084 | } |
| 1085 | p, err = hostfiles.NewFromCopy(files, prefix, emptyManifest(), nil) |
| 1086 | if err != nil { |
| 1087 | return err |
| 1088 | } |
| 1089 | p.SetOrigins(own) |
| 1090 | default: |
| 1091 | // The chain is the holder's: the newest authorized manifest through |
| 1092 | // it wins, a recovery's own included (the prepared rollback |
| 1093 | // generation's, or a later recovery publication's). Only when that |
| 1094 | // chain holds no manifest, the first recovery admission, whose |
| 1095 | // predecessor is the target's generation in the other region, does |
| 1096 | // src_gen bridge the missing source baseline (item 3 as re-ruled; |
| 1097 | // the database selector's distinction, movesrecoveryowners.go |
| 1098 | // baseline: the recorded source restore only while the recovery has |
| 1099 | // no root of its own). |
| 1100 | base, origins, found, err := f.newestManifest(ctx, o, in.Admission, h.Location, movesFilesChain(h, f.HomeRegion, f.Region)) |
| 1101 | if err != nil { |
| 1102 | return err |
| 1103 | } |
| 1104 | if !found && restoresSourceBaseline(o.Claim) { |
| 1105 | src, _ := recoverySourceGeneration(o.Claim) |
| 1106 | base, origins, found, err = f.newestManifest(ctx, o, in.Admission, h.Location, movesFilesChainFrom(h, src, f.HomeRegion, f.Region)) |
| 1107 | if err != nil { |
| 1108 | return err |
| 1109 | } |
| 1110 | } |
| 1111 | if !found { |
| 1112 | never := false |
| 1113 | if f.Evidence != nil && f.Seed == nil { |
| 1114 | pin, ok, err := f.Evidence(ctx, o) |
| 1115 | if err != nil { |
| 1116 | return fmt.Errorf("host: the claim pin's files evidence: %w", err) |
| 1117 | } |
| 1118 | never = ok && pin.NeverPublished() |
| 1119 | } |
| 1120 | if !never && f.Seed == nil { |
| 1121 | return fmt.Errorf("%w: no files manifest in the admitted predecessor chain of %s (%s)", hostfiles.ErrNoEvidence, o.Host, h.Generation) |
| 1122 | } |
| 1123 | own, err := f.proveOrigins(ctx, o, in.Admission, h.Location, []string{gen}) |
| 1124 | if err != nil { |
| 1125 | return err |
| 1126 | } |
| 1127 | base, origins = emptyManifest(), own |
| 1128 | } |
| 1129 | p, err = hostfiles.NewFromCopy(files, prefix, base, origins) |
| 1130 | if err != nil { |
| 1131 | return err |
| 1132 | } |
| 1133 | p.SetOrigins(origins) |
| 1134 | } |
| 1135 | if f.Seed != nil { |
| 1136 | if err := f.Seed.stage(ctx, o, in, work, p, envFile); err != nil { |
| 1137 | return err |
| 1138 | } |
| 1139 | } |
| 1140 | if err := p.StagePlatform(ctx, hostfiles.EnvItem, env); err != nil { |
| 1141 | return err |
| 1142 | } |
| 1143 | if err := p.StagePlatformDelete(instancekey.AdoptionItem); err != nil { |
| 1144 | return err |
| 1145 | } |
| 1146 | if f.Seed != nil { |
| 1147 | if err := f.Seed.check(ctx, o, in); err != nil { |
| 1148 | return err |
| 1149 | } |
| 1150 | } |
| 1151 | if _, err := p.Commit(ctx); err != nil { |
| 1152 | return err |
| 1153 | } |
| 1154 | return work.With(ctx, in.Identity, func() error { |
| 1155 | return removeClaimAdoption(envFile, o.Host+instancekey.AdoptionSuffix) |
| 1156 | }) |
| 1157 | } |
| 1158 | |
| 1159 | // readClaimEnvFile reads the env file's exact bytes: in a nested instance |
| 1160 | // directory, which the tenant can write, only beneath it (readInstanceFile: |
| 1161 | // no link at any component, a regular file, at most the env read limit), |
| 1162 | // and a missing file is an error either way. |
| 1163 | func readClaimEnvFile(e claimEnvFile) ([]byte, error) { |
| 1164 | if !e.Nested { |
| 1165 | return os.ReadFile(e.Path()) |
| 1166 | } |
| 1167 | env, absent, err := readInstanceFile(e.Dir, e.Name, instancekey.EnvReadLimit) |
| 1168 | if err == nil && absent { |
| 1169 | err = fmt.Errorf("%s: %w", e.Path(), fs.ErrNotExist) |
| 1170 | } |
| 1171 | return env, err |
| 1172 | } |
| 1173 | |
| 1174 | // removeClaimAdoption removes the adoption record beside the env file, a |
| 1175 | // nested one beneath the instance directory (instancefs.RemoveBeneath). A |
| 1176 | // record that is not there is already removed. |
| 1177 | func removeClaimAdoption(e claimEnvFile, name string) error { |
| 1178 | var err error |
| 1179 | if e.Nested { |
| 1180 | err = instancefs.RemoveBeneath(e.Dir, name) |
| 1181 | } else { |
| 1182 | err = os.Remove(filepath.Join(e.Dir, name)) |
| 1183 | } |
| 1184 | if err != nil && !errors.Is(err, fs.ErrNotExist) { |
| 1185 | return err |
| 1186 | } |
| 1187 | return nil |
| 1188 | } |
| 1189 | |
amadan