carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package moves
2
3import (
4 "bytes"
5 "context"
6 "crypto/sha256"
7 "encoding/hex"
8 "encoding/json"
9 "errors"
10 "fmt"
11 "io"
12 "log/slog"
13 "os"
14 "path/filepath"
15 "strings"
16 "time"
17
18 "github.com/benbjohnson/litestream"
19 "github.com/carlosframework/platform/internal/hostclaim"
20 "github.com/carlosframework/platform/internal/lease"
21 "github.com/superfly/ltx"
22)
23
24// Errors a build refuses with. Both are internal: Task 10's landing decides
25// what an operator reads.
26var (
27 // ErrBuildIncomplete: the pinned set does not restore to exactly N (a
28 // gap between files, or a set that stops short). Nothing was written.
29 ErrBuildIncomplete = errors.New("moves: the sealed replica does not restore to the required position")
30 // ErrBuildDiverged: an object already under one of the generation's
31 // keys is not what this build would write (another intent, other
32 // snapshot or anchor bytes, another manifest). The generation is new
33 // and the build is deterministic, so this is never a replay.
34 ErrBuildDiverged = errors.New("moves: the prepared generation holds an object this build did not make")
35 // ErrBuildUnpinned: a version id the build would record or read back
36 // pins nothing ("" is no version; "null" is what a bucket whose
37 // versioning is suspended answers, and the next write to that key
38 // replaces it). Every artifact identity the manifest names, and every
39 // current version a replay chooses, must be immutable, so the build
40 // fails before reading or recording such an id and no install.json is
41 // written (G27 on the output side).
42 ErrBuildUnpinned = errors.New("moves: a prepared generation object has no usable version id")
43)
44
45// pinned is the check every recorded or chosen version id passes.
46func pinned(key, versionID string) error {
47 if !lease.UsableVersionID(versionID) {
48 return fmt.Errorf("%w: %s as %q", ErrBuildUnpinned, key, versionID)
49 }
50 return nil
51}
52
53// buildContractVersion is the V of the intent and the manifest.
54const buildContractVersion = 1
55
56// BuildIntent is written before any other PUT with the encoding inputs,
57// so a replay encodes identical bytes (§7.6 building, R7-4). Timestamp is
58// the LTX header timestamp in milliseconds since the epoch.
59type BuildIntent struct {
60 V int `json:"v"`
61 Op string `json:"op"`
62 Host string `json:"host"`
63 N uint64 `json:"snapshot_txid"`
64 PageSize uint32 `json:"page_size"`
65 Commit uint32 `json:"commit"`
66 Timestamp int64 `json:"timestamp"`
67}
68
69// GenerationPrefix is `<location>/g-<op>/`: the prepared generation a move
70// (or a relocation) installs under a holder's location. The location is
71// normalised with TrimSuffix(location, "/") + "/" through ReplicaSource,
72// whose rule this shares with hostbox.GenerationPrefix and genselect's
73// prefix; the four change together (G24, I73).
74func GenerationPrefix(location, op string) string {
75 prefix, _ := ReplicaSource(hostclaim.Holder{Location: location, Generation: op})
76 return prefix
77}
78
79// FilesGenerationBase is `<location>/files/g-`, the start of every
80// generation's files prefix in a location (GenerationFilesPrefix): what
81// follows it up to the next "/" is the generation.
82func FilesGenerationBase(location string) string {
83 return strings.TrimSuffix(location, "/") + "/files/g-"
84}
85
86// GenerationFilesPrefix is `<location>/files/g-<generation>/` (spec §7.6):
87// a generation's files manifest and the content objects it published. It
88// is the one files prefix rule, GenerationPrefix's twin: the C8 files
89// provider (host.movesFilesPrefix), the park view's files live set and
90// classifier, and region HA's relocation (its pinner, build, install,
91// release flush and probe) all name a generation's files through it.
92func GenerationFilesPrefix(location, generation string) string {
93 return FilesGenerationBase(location) + generation + "/"
94}
95
96// SnapshotKey is the generation's level-9 snapshot [1, n].
97func SnapshotKey(prefix string, n uint64) string {
98 return prefix + fmt.Sprintf("%04x/", litestream.SnapshotLevel) + ltx.FormatFilename(1, ltx.TXID(n))
99}
100
101// AnchorKey is the generation's level-0 anchor [n, n].
102func AnchorKey(prefix string, n uint64) string {
103 return prefix + "0000/" + ltx.FormatFilename(ltx.TXID(n), ltx.TXID(n))
104}
105
106// BuildIntentKey is where the intent lives in the generation.
107func BuildIntentKey(prefix string) string { return prefix + "build-intent.json" }
108
109// InstallManifestKey is where the completed install manifest lives.
110func InstallManifestKey(prefix string) string { return prefix + "install.json" }
111
112// BuildInputs are one generation build. Source is the pinned version set
113// (a move: the seal's replica/ objects; region HA's relocation: the
114// versions a genselect plan or a current listing pins, L16); the
115// SealedClient is built over it, so nothing here names a staging lane.
116// WorkDir is the build's scratch, `<JournalDir>/build/` (G34): the
117// restored database is left there as `<WorkDir>/<Host>.db` for the
118// landing's `restored` to rename into place on the same filesystem.
119// Verify is the database check; nil means lease.CheckDB. Now is the clock
120// for the intent's timestamp and the manifest's BuiltAt; nil means
121// time.Now.
122type BuildInputs struct {
123 Store lease.VersionedStore
124 Bucket string
125 Source []SealedObject
126 N uint64
127 GenerationPrefix string
128 Op, Host string
129 WorkDir string
130 Now func() time.Time
131 Verify func(dbPath string) error
132}
133
134// buildName refuses a host or op that is empty or could leave WorkDir:
135// the output path is built from the host and both name the generation.
136func buildName(kind, s string) error {
137 if s == "" || s == "." || s == ".." || strings.ContainsAny(s, "/\\\x00") {
138 return fmt.Errorf("moves: build has an unusable %s %q", kind, s)
139 }
140 return nil
141}
142
143func (in BuildInputs) validate() error {
144 if in.Store == nil || in.Bucket == "" {
145 return errors.New("moves: build needs a store and a bucket")
146 }
147 if in.N == 0 {
148 return errors.New("moves: build needs a position")
149 }
150 if in.GenerationPrefix == "" || !strings.HasSuffix(in.GenerationPrefix, "/") {
151 return fmt.Errorf("moves: build needs a generation prefix ending in /, not %q", in.GenerationPrefix)
152 }
153 if err := buildName("op", in.Op); err != nil {
154 return err
155 }
156 if err := buildName("host", in.Host); err != nil {
157 return err
158 }
159 if in.WorkDir == "" {
160 return errors.New("moves: build needs a work directory")
161 }
162 if len(in.Source) == 0 {
163 return fmt.Errorf("%w: the pinned set is empty", ErrBuildIncomplete)
164 }
165 return nil
166}
167
168// BuildGeneration builds the prepared generation from the pinned set
169// (§5.1 steps 2 and 7, §7.6 building): litestream's own planner and
170// restore over a SealedClient into `<WorkDir>/<Host>.db`, refused unless
171// the plan ends exactly at N and every planned file verifies; the database
172// check; then, in order, the intent, the snapshot [1, N], the anchor
173// [N, N] and the completed InstallManifest, each create-only. A replay at
174// any boundary restores again from the same versions (the scratch file is
175// removed first: litestream refuses an existing output and the file is
176// reproducible), reads the intent it finds and so encodes identical bytes,
177// and reuses an object it finds under its key only when the bytes are
178// equal. The build never deletes an object.
179func BuildGeneration(ctx context.Context, in BuildInputs) (InstallManifest, error) {
180 if err := in.validate(); err != nil {
181 return InstallManifest{}, err
182 }
183 now := in.Now
184 if now == nil {
185 now = time.Now
186 }
187 verify := in.Verify
188 if verify == nil {
189 verify = lease.CheckDB
190 }
191 logger := slog.New(slog.NewTextHandler(io.Discard, nil))
192
193 client, err := NewSealedClient(in.Store, in.Bucket, in.Source)
194 if err != nil {
195 return InstallManifest{}, err
196 }
197 target := ltx.TXID(in.N)
198
199 // The plan: litestream's own, rooted at the newest snapshot at or
200 // below N and extended file by file. A gap under a TXID target is
201 // ErrTxNotAvailable (the contiguity error is only raised for a
202 // latest-state plan), as is a set that stops short.
203 plan, err := litestream.CalcRestorePlan(ctx, client, target, time.Time{}, logger)
204 if err != nil {
205 if errors.Is(err, litestream.ErrTxNotAvailable) || errors.Is(err, litestream.ErrNoSnapshots) {
206 return InstallManifest{}, fmt.Errorf("%w: planning to %s: %v", ErrBuildIncomplete, target, err)
207 }
208 return InstallManifest{}, fmt.Errorf("moves: plan the restore: %w", err)
209 }
210 if len(plan) == 0 || plan[len(plan)-1].MaxTXID != target {
211 return InstallManifest{}, fmt.Errorf("%w: the plan ends at %s, not %s", ErrBuildIncomplete, plan[len(plan)-1].MaxTXID, target)
212 }
213 for _, info := range plan {
214 if err := verifySealedFile(ctx, client, info); err != nil {
215 return InstallManifest{}, err
216 }
217 }
218
219 // A fresh scratch file. Litestream refuses an existing output path
220 // and writes through `<path>.tmp`; a crash can leave either, plus the
221 // sidecars the database check attaches.
222 if err := os.MkdirAll(in.WorkDir, 0o700); err != nil {
223 return InstallManifest{}, fmt.Errorf("moves: build scratch: %w", err)
224 }
225 dbPath := filepath.Join(in.WorkDir, in.Host+".db")
226 for _, p := range []string{dbPath, dbPath + ".tmp", dbPath + "-wal", dbPath + "-shm", dbPath + "-txid"} {
227 if err := os.Remove(p); err != nil && !errors.Is(err, os.ErrNotExist) {
228 return InstallManifest{}, fmt.Errorf("moves: clear build scratch: %w", err)
229 }
230 }
231 opt := litestream.NewRestoreOptions()
232 opt.OutputPath = dbPath
233 opt.TXID = target
234 if err := litestream.NewReplicaWithClient(nil, client).Restore(ctx, opt); err != nil {
235 if errors.Is(err, litestream.ErrTxNotAvailable) || errors.Is(err, litestream.ErrNoSnapshots) {
236 return InstallManifest{}, fmt.Errorf("%w: restoring to %s: %v", ErrBuildIncomplete, target, err)
237 }
238 return InstallManifest{}, fmt.Errorf("moves: restore to %s: %w", target, err)
239 }
240
241 pageSize, pageCount, err := ReadDBHeader(dbPath)
242 if err != nil {
243 return InstallManifest{}, fmt.Errorf("moves: restored database: %w", err)
244 }
245 st, err := os.Stat(dbPath)
246 if err != nil {
247 return InstallManifest{}, err
248 }
249 if st.Size() != int64(pageSize)*int64(pageCount) {
250 return InstallManifest{}, fmt.Errorf("moves: restored database is %d bytes, its header says %d pages of %d", st.Size(), pageCount, pageSize)
251 }
252 if err := verify(dbPath); err != nil {
253 return InstallManifest{}, fmt.Errorf("moves: restored database check: %w", err)
254 }
255 // The check may attach a WAL and a shared-memory file; neither is part
256 // of the build's result.
257 for _, p := range []string{dbPath + "-wal", dbPath + "-shm"} {
258 _ = os.Remove(p)
259 }
260
261 // The intent, before any other PUT. On a replay the existing intent's
262 // timestamp is the one that gives identical bytes.
263 intent := BuildIntent{V: buildContractVersion, Op: in.Op, Host: in.Host, N: in.N, PageSize: pageSize, Commit: pageCount, Timestamp: now().UnixMilli()}
264 intent, err = putOrReadIntent(ctx, in, intent)
265 if err != nil {
266 return InstallManifest{}, err
267 }
268
269 f, err := os.Open(dbPath)
270 if err != nil {
271 return InstallManifest{}, err
272 }
273 defer f.Close()
274
275 snapshot, snapshotCRC, err := EncodeFull(f, pageSize, pageCount, 1, target, intent.Timestamp, 0)
276 if err != nil {
277 return InstallManifest{}, err
278 }
279 snapshotKey := SnapshotKey(in.GenerationPrefix, in.N)
280 snapshotVersion, err := putOrReuse(ctx, in, snapshotKey, snapshot)
281 if err != nil {
282 return InstallManifest{}, err
283 }
284 snapshot = nil
285
286 anchor, anchorCRC, err := EncodeFull(f, pageSize, pageCount, target, target, intent.Timestamp, litestream.WALHeaderSize)
287 if err != nil {
288 return InstallManifest{}, err
289 }
290 anchorKey := AnchorKey(in.GenerationPrefix, in.N)
291 anchorVersion, err := putOrReuse(ctx, in, anchorKey, anchor)
292 if err != nil {
293 return InstallManifest{}, err
294 }
295 anchor = nil
296
297 h := sha256.New()
298 if _, err := io.Copy(h, io.NewSectionReader(f, 0, st.Size())); err != nil {
299 return InstallManifest{}, fmt.Errorf("moves: hash the restored database: %w", err)
300 }
301
302 m := InstallManifest{
303 V: buildContractVersion, Op: in.Op, Host: in.Host, N: in.N,
304 SnapshotKey: snapshotKey, SnapshotVersion: snapshotVersion, SnapshotCRC64: snapshotCRC,
305 AnchorKey: anchorKey, AnchorVersion: anchorVersion, AnchorCRC64: anchorCRC,
306 IntegrityCheck: "ok",
307 DBSHA256: hex.EncodeToString(h.Sum(nil)),
308 BuiltAt: now().UTC(),
309 }
310 return putOrReadManifest(ctx, in, m)
311}
312
313// verifySealedFile reads one planned file whole through the sealed client
314// and checks its CRC-64 trailer and that its header is the position the
315// seal named it by.
316func verifySealedFile(ctx context.Context, client *SealedClient, info *ltx.FileInfo) error {
317 rc, err := client.OpenLTXFile(ctx, info.Level, info.MinTXID, info.MaxTXID, 0, 0)
318 if err != nil {
319 return fmt.Errorf("moves: open sealed %s: %w", ltx.FormatFilename(info.MinTXID, info.MaxTXID), err)
320 }
321 defer rc.Close()
322 dec := ltx.NewDecoder(rc)
323 if err := dec.Verify(); err != nil {
324 return fmt.Errorf("moves: sealed %s at level %d does not verify: %w", ltx.FormatFilename(info.MinTXID, info.MaxTXID), info.Level, err)
325 }
326 if h := dec.Header(); h.MinTXID != info.MinTXID || h.MaxTXID != info.MaxTXID {
327 return fmt.Errorf("moves: sealed %s at level %d carries [%s,%s]", ltx.FormatFilename(info.MinTXID, info.MaxTXID), info.Level, h.MinTXID, h.MaxTXID)
328 }
329 return nil
330}
331
332// currentVersion is the object version currently under key, found by an
333// exact-key listing (a store names versions, never a bare key). Its id
334// must pin an immutable object before anything is read by it.
335func currentVersion(ctx context.Context, in BuildInputs, key string) (lease.ObjectVersion, error) {
336 vs, err := in.Store.ListVersions(ctx, in.Bucket, key)
337 if err != nil {
338 return lease.ObjectVersion{}, fmt.Errorf("moves: list %s: %w", key, err)
339 }
340 for _, v := range vs {
341 if v.Key == key && v.IsLatest && !v.DeleteMarker {
342 if err := pinned(key, v.VersionID); err != nil {
343 return lease.ObjectVersion{}, err
344 }
345 return v, nil
346 }
347 }
348 return lease.ObjectVersion{}, fmt.Errorf("moves: %s exists but has no current version", key)
349}
350
351// putOrReuse writes body create-only under key and returns the version id.
352// When the key already holds an object, the current version's bytes must
353// equal body (identical by construction from the intent) and its id is the
354// answer; anything else is ErrBuildDiverged.
355func putOrReuse(ctx context.Context, in BuildInputs, key string, body []byte) (string, error) {
356 v, err := in.Store.PutIfAbsent(ctx, in.Bucket, key, body)
357 if err == nil {
358 if err := pinned(key, v); err != nil {
359 return "", err
360 }
361 return v, nil
362 }
363 if !errors.Is(err, lease.ErrExists) {
364 return "", fmt.Errorf("moves: put %s: %w", key, err)
365 }
366 cur, err := currentVersion(ctx, in, key)
367 if err != nil {
368 return "", err
369 }
370 have, err := in.Store.GetVersion(ctx, in.Bucket, key, cur.VersionID)
371 if err != nil {
372 return "", fmt.Errorf("moves: read %s@%s: %w", key, cur.VersionID, err)
373 }
374 if !bytes.Equal(have, body) {
375 return "", fmt.Errorf("%w: %s@%s", ErrBuildDiverged, key, cur.VersionID)
376 }
377 return cur.VersionID, nil
378}
379
380// putOrReadIntent writes the intent create-only, or reads the one already
381// there: it must name the same op, host, position, page size and commit,
382// and its timestamp is the one this build encodes with.
383func putOrReadIntent(ctx context.Context, in BuildInputs, want BuildIntent) (BuildIntent, error) {
384 key := BuildIntentKey(in.GenerationPrefix)
385 body, err := json.Marshal(want)
386 if err != nil {
387 return BuildIntent{}, err
388 }
389 if v, err := in.Store.PutIfAbsent(ctx, in.Bucket, key, body); err == nil {
390 if err := pinned(key, v); err != nil {
391 return BuildIntent{}, err
392 }
393 return want, nil
394 } else if !errors.Is(err, lease.ErrExists) {
395 return BuildIntent{}, fmt.Errorf("moves: put %s: %w", key, err)
396 }
397 cur, err := currentVersion(ctx, in, key)
398 if err != nil {
399 return BuildIntent{}, err
400 }
401 have, err := in.Store.GetVersion(ctx, in.Bucket, key, cur.VersionID)
402 if err != nil {
403 return BuildIntent{}, fmt.Errorf("moves: read %s@%s: %w", key, cur.VersionID, err)
404 }
405 var got BuildIntent
406 if err := json.Unmarshal(have, &got); err != nil {
407 return BuildIntent{}, fmt.Errorf("%w: %s@%s is not an intent: %v", ErrBuildDiverged, key, cur.VersionID, err)
408 }
409 if got.V != want.V || got.Op != want.Op || got.Host != want.Host || got.N != want.N || got.PageSize != want.PageSize || got.Commit != want.Commit {
410 return BuildIntent{}, fmt.Errorf("%w: %s@%s names other inputs", ErrBuildDiverged, key, cur.VersionID)
411 }
412 return got, nil
413}
414
415// putOrReadManifest writes the completed manifest create-only, or reads
416// the one already there, which must name the same position, keys,
417// versions, checksums and database digest; the stored one (its BuiltAt
418// included) is the answer.
419func putOrReadManifest(ctx context.Context, in BuildInputs, want InstallManifest) (InstallManifest, error) {
420 key := InstallManifestKey(in.GenerationPrefix)
421 body, err := json.Marshal(want)
422 if err != nil {
423 return InstallManifest{}, err
424 }
425 if v, err := in.Store.PutIfAbsent(ctx, in.Bucket, key, body); err == nil {
426 if err := pinned(key, v); err != nil {
427 return InstallManifest{}, err
428 }
429 return want, nil
430 } else if !errors.Is(err, lease.ErrExists) {
431 return InstallManifest{}, fmt.Errorf("moves: put %s: %w", key, err)
432 }
433 cur, err := currentVersion(ctx, in, key)
434 if err != nil {
435 return InstallManifest{}, err
436 }
437 have, err := in.Store.GetVersion(ctx, in.Bucket, key, cur.VersionID)
438 if err != nil {
439 return InstallManifest{}, fmt.Errorf("moves: read %s@%s: %w", key, cur.VersionID, err)
440 }
441 var got InstallManifest
442 if err := json.Unmarshal(have, &got); err != nil {
443 return InstallManifest{}, fmt.Errorf("%w: %s@%s is not a manifest: %v", ErrBuildDiverged, key, cur.VersionID, err)
444 }
445 same := got.V == want.V && got.Op == want.Op && got.Host == want.Host && got.N == want.N &&
446 got.SnapshotKey == want.SnapshotKey && got.SnapshotVersion == want.SnapshotVersion && got.SnapshotCRC64 == want.SnapshotCRC64 &&
447 got.AnchorKey == want.AnchorKey && got.AnchorVersion == want.AnchorVersion && got.AnchorCRC64 == want.AnchorCRC64 &&
448 got.IntegrityCheck == want.IntegrityCheck && got.DBSHA256 == want.DBSHA256
449 if !same {
450 return InstallManifest{}, fmt.Errorf("%w: %s@%s records another build", ErrBuildDiverged, key, cur.VersionID)
451 }
452 return got, nil
453}
454