| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "crypto/sha256" |
| 6 | "encoding/hex" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/genselect" |
| 14 | "github.com/carlosframework/platform/internal/hostbox" |
| 15 | "github.com/carlosframework/platform/internal/hostclaim" |
| 16 | "github.com/carlosframework/platform/internal/hostfiles" |
| 17 | "github.com/carlosframework/platform/internal/hostrelocate" |
| 18 | "github.com/carlosframework/platform/internal/lease" |
| 19 | "github.com/carlosframework/platform/internal/moves" |
| 20 | "github.com/carlosframework/platform/internal/release" |
| 21 | ) |
| 22 | |
| 23 | // relocationBuild is the target's build at installing: its guard |
| 24 | // activated at install_stamp from an empty box (ruling 43), then P built |
| 25 | // under the replica lease with every durable write behind the build |
| 26 | // authority (ruling 59), then the build's host-named scratch removed |
| 27 | // (ruling 44). |
| 28 | func (a ClaimAssembly) relocationBuild(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed) error { |
| 29 | activate := a.relocationActivate(cfg) |
| 30 | build := a.relocationBuildP(cfg) |
| 31 | return func(ctx context.Context, o hostclaim.Observed) error { |
| 32 | captured, err := activate(ctx, o) |
| 33 | if err != nil { |
| 34 | return err |
| 35 | } |
| 36 | if err := build(ctx, o, a.relocationBuildAuthority(cfg, o, captured)); err != nil { |
| 37 | return err |
| 38 | } |
| 39 | return removeRelocationScratch(cfg, o.Claim.Op.ID) |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | // relocationBuildP builds P from the journaled pin (§5.8 "Building the |
| 44 | // prepared generation", L16, L17) under the replica lease on |
| 45 | // holder.location, taken for the roster's cloud region (ruling 51); the |
| 46 | // lease's loss cancels what follows. auth is the build authority, run |
| 47 | // before the manifest copy, before the database build, and before every |
| 48 | // write the builder makes (authorityStore). P's files manifest is written |
| 49 | // first, so a crash between the two steps leaves an install manifest never |
| 50 | // ahead of its files manifest. A replay from installing finds its own |
| 51 | // manifest copy (CopyPinnedManifest is create-exact) and its own intent, |
| 52 | // snapshot and anchor (moves §7.6). |
| 53 | func (a ClaimAssembly) relocationBuildP(cfg RelocationAssemblyConfig) func(context.Context, hostclaim.Observed, func(context.Context) error) error { |
| 54 | return func(ctx context.Context, o hostclaim.Observed, auth func(context.Context) error) error { |
| 55 | if cfg.Versions == nil || cfg.Data == nil || cfg.Bucket == "" || cfg.LeaseRegion == "" || auth == nil || o.Claim == nil || o.Claim.Holder == nil { |
| 56 | return ErrClaimStopUnavailable |
| 57 | } |
| 58 | c := o.Claim |
| 59 | var ref hostclaim.RecoveryPin |
| 60 | if err := json.Unmarshal([]byte(c.Op.Journal["pin"]), &ref); err != nil { |
| 61 | return err |
| 62 | } |
| 63 | pin, err := hostrelocate.ReadPin(ctx, a.Claims, ref) |
| 64 | if err != nil { |
| 65 | return err |
| 66 | } |
| 67 | p := c.Op.Journal["prepared"] |
| 68 | work, err := relocationBuildDir(cfg, c.Op.ID) |
| 69 | if err != nil || p == "" { |
| 70 | return errors.Join(fmt.Errorf("%w: no prepared generation to build", hostrelocate.ErrRefused), err) |
| 71 | } |
| 72 | // The authority first: a stale build never writes the lease object |
| 73 | // on holder.location, where it would refuse the source's own wake. |
| 74 | if err := auth(ctx); err != nil { |
| 75 | return err |
| 76 | } |
| 77 | acquire := a.acquireReplicaLease |
| 78 | if acquire == nil { |
| 79 | acquire = func(ctx context.Context, bucket, prefix, region string) (replicaLease, error) { |
| 80 | return lease.Acquire(ctx, bucket, prefix, region) |
| 81 | } |
| 82 | } |
| 83 | held, err := acquire(ctx, cfg.Bucket, c.Holder.Location, cfg.LeaseRegion) |
| 84 | if err != nil { |
| 85 | return err |
| 86 | } |
| 87 | defer func() { _ = held.Release(context.WithoutCancel(ctx)) }() |
| 88 | alive, stop := context.WithCancel(ctx) |
| 89 | defer stop() |
| 90 | go func() { |
| 91 | select { |
| 92 | case <-held.Lost(): |
| 93 | stop() |
| 94 | case <-alive.Done(): |
| 95 | } |
| 96 | }() |
| 97 | check := func() error { |
| 98 | if err := auth(alive); err != nil { |
| 99 | return err |
| 100 | } |
| 101 | return alive.Err() |
| 102 | } |
| 103 | if err := check(); err != nil { |
| 104 | return err |
| 105 | } |
| 106 | if _, err := hostrelocate.CopyPinnedManifest(alive, a.Claims, cfg.Data, pin.Files, moves.GenerationFilesPrefix(c.Holder.Location, p)); err != nil { |
| 107 | return err |
| 108 | } |
| 109 | if pin.DB == nil { |
| 110 | return nil // an evidenced empty bootstrap: no snapshot, no anchor (claim P6-D3 forbids [1, 0]) |
| 111 | } |
| 112 | if err := check(); err != nil { |
| 113 | return err |
| 114 | } |
| 115 | src := make([]moves.SealedObject, 0, len(pin.DB.Files)) |
| 116 | for _, f := range pin.DB.Files { |
| 117 | src = append(src, moves.SealedObject{Key: f.Key, VersionID: f.VersionID, Size: f.Size}) |
| 118 | } |
| 119 | if err := os.MkdirAll(work, 0o700); err != nil { |
| 120 | return err |
| 121 | } |
| 122 | _, err = moves.BuildGeneration(alive, moves.BuildInputs{Store: authorityStore{VersionedStore: cfg.Versions, check: check}, Bucket: cfg.Bucket, Source: src, N: pin.DB.MaxTXID, |
| 123 | GenerationPrefix: moves.GenerationPrefix(c.Holder.Location, p), Op: c.Op.ID, Host: o.Host, WorkDir: work, Now: cfg.Now, Verify: lease.CheckDB}) |
| 124 | return err |
| 125 | } |
| 126 | } |
| 127 | |
| 128 | // replicaLease is the part of *lease.Held the build uses: its loss and |
| 129 | // its release. |
| 130 | type replicaLease interface { |
| 131 | Lost() <-chan struct{} |
| 132 | Release(context.Context) error |
| 133 | } |
| 134 | |
| 135 | // authorityStore is the versioned store the builder writes through: every |
| 136 | // write re-checks the build's authority first, so a delayed build whose |
| 137 | // phase, executor, stamp, execution lease or replica lease moved writes |
| 138 | // nothing more. Reads are the store's own; the build never deletes. |
| 139 | type authorityStore struct { |
| 140 | lease.VersionedStore |
| 141 | check func() error |
| 142 | } |
| 143 | |
| 144 | func (s authorityStore) PutIfAbsent(ctx context.Context, bucket, key string, body []byte) (string, error) { |
| 145 | if err := s.check(); err != nil { |
| 146 | return "", err |
| 147 | } |
| 148 | return s.VersionedStore.PutIfAbsent(ctx, bucket, key, body) |
| 149 | } |
| 150 | |
| 151 | func (s authorityStore) CopyVersion(ctx context.Context, sb, sk, sv, db, dk string, createOnly bool) (string, string, error) { |
| 152 | if err := s.check(); err != nil { |
| 153 | return "", "", err |
| 154 | } |
| 155 | return s.VersionedStore.CopyVersion(ctx, sb, sk, sv, db, dk, createOnly) |
| 156 | } |
| 157 | |
| 158 | func (s authorityStore) DeleteVersion(context.Context, string, string, string) error { |
| 159 | return fmt.Errorf("%w: the build never deletes", hostrelocate.ErrRefused) |
| 160 | } |
| 161 | |
| 162 | // relocationBuildDir is the build's scratch for op, moves.BuildInputs' |
| 163 | // WorkDir: <WorkDir>/<op>, where BuildGeneration leaves <host>.db. |
| 164 | func relocationBuildDir(cfg RelocationAssemblyConfig, op string) (string, error) { |
| 165 | if cfg.WorkDir == "" || !validOpID(op) { |
| 166 | return "", ErrClaimStopUnavailable |
| 167 | } |
| 168 | return filepath.Join(cfg.WorkDir, op), nil |
| 169 | } |
| 170 | |
| 171 | // removeRelocationScratch removes the build's scratch for op, which names |
| 172 | // the host (<WorkDir>/<op>/<host>.db). The build removes it once P is |
| 173 | // built; the local install removes it again first, so a crash in between |
| 174 | // never leaves a host-named file for the census. |
| 175 | func removeRelocationScratch(cfg RelocationAssemblyConfig, op string) error { |
| 176 | dir, err := relocationBuildDir(cfg, op) |
| 177 | if err != nil { |
| 178 | return err |
| 179 | } |
| 180 | return os.RemoveAll(dir) |
| 181 | } |
| 182 | |
| 183 | // readInstallManifest is the current install.json of a generation prefix, |
| 184 | // its bytes, and its version: the one reader for the build verifier, the |
| 185 | // prepared marker and the probe driver. No current version is an error |
| 186 | // that is both hostbox.ErrChanged and os.ErrNotExist. |
| 187 | func readInstallManifest(ctx context.Context, data genselect.Reader, bucket, prefix string) (moves.InstallManifest, []byte, error) { |
| 188 | key := moves.InstallManifestKey(prefix) |
| 189 | versions, err := data.ListVersions(ctx, bucket, key) |
| 190 | if err != nil { |
| 191 | return moves.InstallManifest{}, nil, err |
| 192 | } |
| 193 | id := "" |
| 194 | for _, v := range versions { |
| 195 | if v.Key == key && v.IsLatest && !v.DeleteMarker { |
| 196 | id = v.VersionID |
| 197 | } |
| 198 | } |
| 199 | if id == "" { |
| 200 | return moves.InstallManifest{}, nil, fmt.Errorf("%w: %s has no install.json (%w)", hostbox.ErrChanged, prefix, os.ErrNotExist) |
| 201 | } |
| 202 | raw, err := data.GetVersion(ctx, bucket, key, id) |
| 203 | if err != nil { |
| 204 | return moves.InstallManifest{}, nil, err |
| 205 | } |
| 206 | var m moves.InstallManifest |
| 207 | if err := json.Unmarshal(raw, &m); err != nil { |
| 208 | return moves.InstallManifest{}, nil, err |
| 209 | } |
| 210 | return m, raw, nil |
| 211 | } |
| 212 | |
| 213 | // PreparedFromInstallManifest is hostrelocate.Prepared: P is built when a |
| 214 | // fresh read verifies its files manifest (sequence 0, exactly the pinned |
| 215 | // entries) and, unless the pin is empty, its completed install manifest at |
| 216 | // the pin's N whose snapshot and anchor versions exist. |
| 217 | type PreparedFromInstallManifest struct { |
| 218 | Claims release.Store // pin objects and the retained manifest |
| 219 | Files release.Store // files manifests |
| 220 | Data lease.VersionedStore // LTX files and install manifests |
| 221 | Bucket string |
| 222 | } |
| 223 | |
| 224 | var _ hostrelocate.Prepared = PreparedFromInstallManifest{} |
| 225 | |
| 226 | func (v PreparedFromInstallManifest) Verify(ctx context.Context, o hostclaim.Observed) (hostclaim.PreparedAuthority, bool, error) { |
| 227 | if v.Claims == nil || v.Files == nil || v.Data == nil || v.Bucket == "" || o.Claim == nil || o.Claim.Holder == nil { |
| 228 | return hostclaim.PreparedAuthority{}, false, ErrClaimStopUnavailable |
| 229 | } |
| 230 | c := o.Claim |
| 231 | var ref hostclaim.RecoveryPin |
| 232 | if err := json.Unmarshal([]byte(c.Op.Journal["pin"]), &ref); err != nil { |
| 233 | return hostclaim.PreparedAuthority{}, false, err |
| 234 | } |
| 235 | pin, err := hostrelocate.ReadPin(ctx, v.Claims, ref) |
| 236 | if err != nil { |
| 237 | return hostclaim.PreparedAuthority{}, false, err |
| 238 | } |
| 239 | p := c.Op.Journal["prepared"] |
| 240 | raw, _, err := v.Files.Get(ctx, hostfiles.ManifestKey(moves.GenerationFilesPrefix(c.Holder.Location, p))) |
| 241 | if errors.Is(err, os.ErrNotExist) { |
| 242 | return hostclaim.PreparedAuthority{}, false, nil // not built yet |
| 243 | } |
| 244 | if err != nil { |
| 245 | return hostclaim.PreparedAuthority{}, false, err |
| 246 | } |
| 247 | var m hostfiles.Manifest |
| 248 | if err := json.Unmarshal(raw, &m); err != nil || m.V != hostfiles.ManifestVersion || m.Seq != 0 { |
| 249 | return hostclaim.PreparedAuthority{}, false, fmt.Errorf("%w: P's files manifest is not a prepared manifest", hostrelocate.ErrRefused) |
| 250 | } |
| 251 | // ReadPin refuses a files digest without its retained copy, so no |
| 252 | // retained copy here is a host that published no files. |
| 253 | pinned := hostfiles.Manifest{Entries: map[string]hostfiles.Entry{}, Platform: map[string]hostfiles.Entry{}} |
| 254 | if pin.Files.Retained != "" { |
| 255 | // The pinner's retained copy of the pinned bytes, never the source |
| 256 | // prefix (ruling 21): P's entries must be exactly theirs. |
| 257 | pinnedRaw, _, err := v.Claims.Get(ctx, pin.Files.Retained) |
| 258 | if err != nil { |
| 259 | return hostclaim.PreparedAuthority{}, false, err |
| 260 | } |
| 261 | if sum := sha256.Sum256(pinnedRaw); hex.EncodeToString(sum[:]) != pin.Files.SHA256 { |
| 262 | return hostclaim.PreparedAuthority{}, false, fmt.Errorf("%w: the pinned files manifest changed", hostrelocate.ErrRefused) |
| 263 | } |
| 264 | if err := json.Unmarshal(pinnedRaw, &pinned); err != nil { |
| 265 | return hostclaim.PreparedAuthority{}, false, err |
| 266 | } |
| 267 | } |
| 268 | if !sameEntries(m, pinned) { |
| 269 | return hostclaim.PreparedAuthority{}, false, fmt.Errorf("%w: P's files manifest is not the pinned manifest", hostrelocate.ErrRefused) |
| 270 | } |
| 271 | filesSum := sha256.Sum256(raw) |
| 272 | auth := hostclaim.PreparedAuthority{Generation: p, FilesSeq: 0, SourceFilesSeq: pin.Files.Seq, FilesManifestSHA256: hex.EncodeToString(filesSum[:])} |
| 273 | if pin.DB == nil { |
| 274 | return auth, true, nil |
| 275 | } |
| 276 | prefix := moves.GenerationPrefix(c.Holder.Location, p) |
| 277 | im, body, err := readInstallManifest(ctx, v.Data, v.Bucket, prefix) |
| 278 | if errors.Is(err, os.ErrNotExist) { |
| 279 | return hostclaim.PreparedAuthority{}, false, nil // the build has not completed |
| 280 | } |
| 281 | if err != nil { |
| 282 | return hostclaim.PreparedAuthority{}, false, err |
| 283 | } |
| 284 | key := moves.InstallManifestKey(prefix) |
| 285 | if im.Op != c.Op.ID || im.Host != o.Host || im.N != pin.DB.MaxTXID || im.SnapshotKey != moves.SnapshotKey(prefix, im.N) || im.AnchorKey != moves.AnchorKey(prefix, im.N) || |
| 286 | im.SnapshotVersion == "" || im.AnchorVersion == "" || im.IntegrityCheck != "ok" { |
| 287 | return hostclaim.PreparedAuthority{}, false, fmt.Errorf("%w: install manifest %s is not this pin's build of P", hostrelocate.ErrRefused, key) |
| 288 | } |
| 289 | for _, want := range []struct{ key, version string }{{im.SnapshotKey, im.SnapshotVersion}, {im.AnchorKey, im.AnchorVersion}} { |
| 290 | vs, err := v.Data.ListVersions(ctx, v.Bucket, want.key) |
| 291 | if err != nil { |
| 292 | return hostclaim.PreparedAuthority{}, false, err |
| 293 | } |
| 294 | found := false |
| 295 | for _, ov := range vs { |
| 296 | if ov.Key == want.key && ov.VersionID == want.version && !ov.DeleteMarker { |
| 297 | found = true |
| 298 | } |
| 299 | } |
| 300 | if !found { |
| 301 | return hostclaim.PreparedAuthority{}, false, fmt.Errorf("%w: %s@%s named by the install manifest is not there", hostrelocate.ErrRefused, want.key, want.version) |
| 302 | } |
| 303 | } |
| 304 | sum := sha256.Sum256(body) |
| 305 | auth.InstallManifest, auth.SHA256, auth.SnapshotTXID = key, hex.EncodeToString(sum[:]), im.N |
| 306 | return auth, true, nil |
| 307 | } |
| 308 | |
| 309 | // sameEntries compares two manifests' entries and platform items, nothing |
| 310 | // else: P's prepared copy differs from the pinned bytes only in Seq. |
| 311 | func sameEntries(a, b hostfiles.Manifest) bool { |
| 312 | type entries struct{ E, P map[string]hostfiles.Entry } |
| 313 | norm := func(m map[string]hostfiles.Entry) map[string]hostfiles.Entry { |
| 314 | if m == nil { |
| 315 | return map[string]hostfiles.Entry{} |
| 316 | } |
| 317 | return m |
| 318 | } |
| 319 | ra, _ := json.Marshal(entries{norm(a.Entries), norm(a.Platform)}) |
| 320 | rb, _ := json.Marshal(entries{norm(b.Entries), norm(b.Platform)}) |
| 321 | return string(ra) == string(rb) |
| 322 | } |
| 323 | |
amadan