carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "io"
9 "log"
10 "os"
11 "path/filepath"
12 "strings"
13 "time"
14
15 "github.com/carlosframework/platform/internal/moves"
16 "github.com/carlosframework/platform/internal/registry"
17 "github.com/carlosframework/platform/internal/release"
18)
19
20// The moves protocol floor on a box (moves spec §11). Once an operator
21// publishes control/moves-protocol.json, no box may activate an executable
22// below its min_release: not by update, not by rollback, not by recovery,
23// not by picking carlos.prev at boot. The floor is the one Phase 1 element
24// that is not behind CARLOS_MOVES_PHASE1 (index D13): publishing the object
25// arms it, and turning the switch off later does not disarm it. A
26// deployment that never published it has none of the files below, and
27// every path here then does what main does.
28//
29// The files, and who writes them:
30//
31// - <DataDir>/.moves/floor.json and the one-line floor.min-release: the
32// floor as this box last read it. Root's apply writes them from the
33// store before it stages anything (armFloor); the agent refreshes them
34// every hour (movesFloorLoop). Both write under .moves/floor.lock, and a
35// floor only rises: a lower published floor is refused, and lowering
36// one is an operator's manual step on each box (floorRegression). Never
37// deleted by this code: a store that no longer answers, or no longer
38// holds the object, leaves them alone. floor.json is the one every Go
39// reader trusts; floor.min-release is for the units (writeFloorFiles).
40// - <optDir>/carlos.prev.version: carlos.prev's release, written by root
41// whenever it replaces carlos.prev (decidePrev), or learned by running
42// `carlos.prev version` when it is missing and a floor is armed.
43// - <optDir>/carlos.prev.allowed, the verdict: the min_release carlos.prev
44// was judged at or above, written only while no registry row is held by
45// an operation (a .prev is a downgrade by definition). judgePrev is the
46// one judge, run by root at every apply and every adopt pass.
47//
48// The two system units choose carlos.prev for a recovery only when the
49// floor line is absent, or equal byte for byte to the verdict. The version
50// comparison happens once, here in Go (moves.ReleaseAtOrAbove, which fails
51// closed); the units only compare two files, through carlos.prev's floor
52// gate (movesfloorgate.go), which never follows a symlink or waits on a
53// FIFO the agent planted.
54//
55// What this is not: a defence against a compromised agent. The agent owns
56// /data and can remove the floor files there. The floor protects the
57// protocol against old builds and honest mistakes, which is what §11 asks.
58
59const movesFloorMaxBytes = 4 << 10
60
61// movesFloorRefreshEvery is the agent's refresh cadence (moves spec §11:
62// a raised floor reaches every box within the hour).
63const movesFloorRefreshEvery = time.Hour
64
65func prevVersionPath(optDir string) string { return filepath.Join(optDir, moves.PrevVersionFile) }
66func prevAllowedPath(optDir string) string { return filepath.Join(optDir, moves.PrevAllowedFile) }
67
68// validFloorRelease is a min_release the floor can be armed at: a release
69// moves.ReleaseAtOrAbove can compare, so nothing that would fail closed on
70// every comparison is ever written to disk.
71func validFloorRelease(v string) bool {
72 ok, err := moves.ReleaseAtOrAbove(v, v)
73 return ok && err == nil
74}
75
76// readLocalFloor reads <dataDir>/.moves/floor.json without following a
77// symlink anywhere below dataDir (root reads it, and the agent owns the
78// tree). (nil, nil) means no floor is armed on this box. A file that is
79// there but cannot be read, or does not hold a usable floor, is an error:
80// the callers treat that as armed and unknown, never as absent.
81func readLocalFloor(dataDir string) (*moves.Floor, error) {
82 f, err := openBeneath(dataDir, filepath.Dir(moves.FloorFile), filepath.Base(moves.FloorFile))
83 if errors.Is(err, os.ErrNotExist) {
84 return nil, nil
85 }
86 if err != nil {
87 return nil, err
88 }
89 defer f.Close()
90 st, err := f.Stat()
91 if err != nil {
92 return nil, err
93 }
94 if !st.Mode().IsRegular() || st.Size() > movesFloorMaxBytes {
95 return nil, fmt.Errorf("%s is not a regular file under %d bytes", f.Name(), movesFloorMaxBytes)
96 }
97 var fl moves.Floor
98 if err := json.NewDecoder(io.LimitReader(f, movesFloorMaxBytes)).Decode(&fl); err != nil {
99 return nil, fmt.Errorf("%s: %w", f.Name(), err)
100 }
101 if !validFloorRelease(fl.MinRelease) {
102 return nil, fmt.Errorf("%s: min_release %q is not a release", f.Name(), fl.MinRelease)
103 }
104 return &fl, nil
105}
106
107// errFloorLockBusy is floor.lock still held by another process after the
108// wait: a writer is active, and may be about to publish a floor. Never a
109// reason to go ahead without the lock (fix round 3): an activation that
110// meets it is deferred.
111var errFloorLockBusy = errors.New("another writer holds the floor lock on this box")
112
113// floorLockWait bounds the wait for floor.lock (movesDir.Lock); a variable
114// so a test can make contention time out quickly.
115var floorLockWait = 2 * time.Minute
116
117// movesDirName is the directory both local floor files live in, below
118// the data dir.
119var movesDirName = filepath.Dir(moves.FloorFile)
120
121// withFloorLock runs fn holding .moves/floor.lock (see floorLockFile). With
122// create, .moves is made when missing (a writer arming the first floor).
123// Without it, a missing .moves means no floor is armed and nothing can be
124// locked: fn runs with a nil dir. An error opening or locking is returned
125// without running fn.
126//
127// Lock order on census boxes: root release exclusion, the apply flock for
128// system updates, carlos.prev lock, participation transaction, moves floor, then
129// registry write lock. A held-stage lock is a leaf. Adoption omits the apply
130// flock. Guarded close and rollback receive the outer release token; they
131// never acquire release exclusion while holding floor or registry locks.
132func withFloorLock(dataDir string, create bool, fn func(d *movesDir) error) error {
133 d, err := openMovesDir(dataDir, create)
134 if errors.Is(err, os.ErrNotExist) && !create {
135 return fn(nil)
136 }
137 if err != nil {
138 return err
139 }
140 defer d.Close()
141 if err := d.Lock(); err != nil {
142 return err
143 }
144 return fn(d)
145}
146
147// writeLocalFloor writes both local floor files under the floor lock.
148func writeLocalFloor(dataDir string, f moves.Floor) error {
149 if !validFloorRelease(f.MinRelease) {
150 return fmt.Errorf("min_release %q is not a release", f.MinRelease)
151 }
152 return withFloorLock(dataDir, true, func(d *movesDir) error { return writeFloorFiles(d, f) })
153}
154
155// writeFloorFiles writes floor.min-release FIRST and floor.json LAST, under
156// the floor lock. Every Go reader (activationAllowed, judgePrev, the edges
157// file) trusts floor.json alone; floor.min-release exists for the units'
158// shell, which can only compare files. So after a crash between the two,
159// floor.min-release can only be AHEAD of floor.json (floors only rise, see
160// floorRegression), and ahead means it matches no verdict judged against
161// the JSON: the units run the current binary, whose own check then uses the
162// JSON. The two can never disagree in the direction that would let the
163// units pick a carlos.prev the Go checks would refuse.
164func writeFloorFiles(d *movesDir, f moves.Floor) error {
165 body, err := json.Marshal(f)
166 if err != nil {
167 return err
168 }
169 if err := d.WriteDurable(filepath.Base(moves.FloorMinReleaseFile), []byte(f.MinRelease+"\n"), 0o644); err != nil {
170 return err
171 }
172 return d.WriteDurable(filepath.Base(moves.FloorFile), append(body, '\n'), 0o644)
173}
174
175// floorRegression says why remote may not replace local, or nil. A floor
176// only rises on a box: a lower protocol minimum, or a min_release that is
177// lower or does not compare, is refused. Lowering a floor is an operator's
178// manual step, box by box: delete <DataDir>/.moves/floor.json and
179// floor.min-release (and publish the lower object), and the next apply or
180// refresh arms what is published.
181func floorRegression(local, remote *moves.Floor) error {
182 if remote.Min < local.Min {
183 return fmt.Errorf("the published floor's protocol minimum %d is below this box's %d; lowering a floor is done by hand on each box", remote.Min, local.Min)
184 }
185 if ok, _ := moves.ReleaseAtOrAbove(remote.MinRelease, local.MinRelease); !ok {
186 return fmt.Errorf("the published min_release %q is below this box's %s; lowering a floor is done by hand on each box", remote.MinRelease, local.MinRelease)
187 }
188 return nil
189}
190
191// errFloorWrite marks a floor the store holds that this box could not
192// write down: the one syncLocalFloor failure that leaves the box enforcing
193// less than what is published.
194var errFloorWrite = errors.New("could not write it on this box")
195
196// floorWriteError is errFloorWrite with what the write said. Its text is
197// what it always was ("could not write it on this box: <cause>"); armFloor
198// reads cause to say the recorded sentence whole (copy record MV-N92).
199type floorWriteError struct{ cause error }
200
201func (e *floorWriteError) Error() string { return fmt.Sprintf("%v: %v", errFloorWrite, e.cause) }
202func (e *floorWriteError) Is(target error) bool { return target == errFloorWrite }
203
204// syncLocalFloor is the rule root's apply and the agent's refresh share:
205// read the published floor and make the local files say the same. It
206// returns the floor on disk afterwards, nil when none is armed.
207//
208// - published, usable and not lower: written, under the floor lock, with
209// the comparison made under the same lock, so an older observation that
210// finishes after a newer one cannot lower the floor;
211// - lower than the floor on disk: refused (floorRegression), logged;
212// - absent from the store: what is on disk stays (once armed, armed);
213// - the store unreadable, or the object unusable: what is on disk stays,
214// and the error is returned to be logged.
215//
216// A local floor.json that cannot be read is replaced by a usable published
217// floor: it holds no value to compare, and the published one is the
218// authority it was copied from.
219func syncLocalFloor(ctx context.Context, s release.Store, dataDir string) (*moves.Floor, error) {
220 if s == nil {
221 local, lerr := readLocalFloor(dataDir)
222 return local, errors.Join(errors.New("no deployment store to read the floor from"), lerr)
223 }
224 remote, rerr := moves.ReadFloor(ctx, s)
225 switch {
226 case rerr != nil:
227 local, lerr := readLocalFloor(dataDir)
228 return local, errors.Join(rerr, lerr)
229 case remote == nil:
230 return readLocalFloor(dataDir)
231 case !validFloorRelease(remote.MinRelease):
232 local, lerr := readLocalFloor(dataDir)
233 return local, errors.Join(fmt.Errorf("the published min_release %q is not a release; the floor on this box stays as it was", remote.MinRelease), lerr)
234 }
235 var out *moves.Floor
236 ran := false
237 err := withFloorLock(dataDir, true, func(d *movesDir) error {
238 ran = true
239 local, lerr := readLocalFloor(dataDir)
240 if lerr == nil && local != nil {
241 if err := floorRegression(local, remote); err != nil {
242 out = local
243 return err
244 }
245 if *local == *remote {
246 if line, ok := readBeneathLine(dataDir, moves.FloorMinReleaseFile); ok && line == remote.MinRelease {
247 out = local
248 return nil
249 }
250 }
251 }
252 if err := writeFloorFiles(d, *remote); err != nil {
253 out, _ = readLocalFloor(dataDir)
254 return &floorWriteError{cause: err}
255 }
256 out = remote
257 return nil
258 })
259 if !ran {
260 out, _ = readLocalFloor(dataDir)
261 return out, &floorWriteError{cause: err}
262 }
263 return out, err
264}
265
266// readBeneathLine reads a one-line file under root without following a
267// symlink below root, trimmed. false means absent, unreadable or empty.
268func readBeneathLine(root, rel string) (string, bool) {
269 f, err := openBeneath(root, strings.Split(filepath.ToSlash(rel), "/")...)
270 if err != nil {
271 return "", false
272 }
273 defer f.Close()
274 if st, err := f.Stat(); err != nil || !st.Mode().IsRegular() {
275 return "", false
276 }
277 return readLineFrom(f)
278}
279
280// readLine reads a one-line file the reader trusts the path of (root's own
281// /opt/carlos, or the agent's own /data), trimmed. false means absent,
282// unreadable or empty.
283func readLine(path string) (string, bool) {
284 f, err := os.Open(path)
285 if err != nil {
286 return "", false
287 }
288 defer f.Close()
289 return readLineFrom(f)
290}
291
292func readLineFrom(r io.Reader) (string, bool) {
293 b, err := io.ReadAll(io.LimitReader(r, 256))
294 if err != nil {
295 return "", false
296 }
297 s := strings.TrimSpace(string(b))
298 return s, s != ""
299}
300
301// writePrevVersion records carlos.prev's release beside it, durably.
302func writePrevVersion(optDir, version string) error {
303 if !validSystemVersion(version) {
304 return fmt.Errorf("carlos.prev: %q is not a version", version)
305 }
306 return writeFileDurable(prevVersionPath(optDir), []byte(version+"\n"), 0o644)
307}
308
309// removeDurable removes path and, when it was there, fsyncs its directory.
310// A path that is already gone is removed.
311func removeDurable(path string) error {
312 err := os.Remove(path)
313 if errors.Is(err, os.ErrNotExist) {
314 return nil
315 }
316 if err != nil {
317 return err
318 }
319 return syncDir(filepath.Dir(path))
320}
321
322// holdFreezer runs fn with the rows an operation holds, held still for
323// fn's length (registry.WithHoldsFrozen's shape).
324type holdFreezer func(fn func(held []registry.Route) error) error
325
326// judgePrev is the one judge of carlos.prev (moves spec §11): it writes the
327// verdict when carlos.prev is at or above the armed floor and no registry
328// row is held by an operation, and removes it otherwise. With no floor
329// armed it touches neither the registry nor carlos.prev, and leaves no
330// verdict (the units do not read one then).
331func judgePrev(optDir, dataDir string, reg *registry.Registry) error {
332 return judgePrevWith(optDir, dataDir, func(fn func([]registry.Route) error) error {
333 if reg == nil {
334 return errors.New("no registry")
335 }
336 return reg.WithHoldsFrozen(fn)
337 }, nil)
338}
339
340// judgePrevWith is judgePrev with its two inputs as functions: freeze gives
341// the held rows, held still while the verdict is written, and learn (nil to
342// skip) answers carlos.prev's version when the sidecar is missing; what it
343// learns is written as the sidecar. Learning runs carlos.prev, which can
344// take seconds, so it happens before any lock is taken.
345//
346// The judgement itself runs under the floor lock with the holds frozen, so
347// the verdict it writes is for the floor and the holds as they stand when
348// it lands. Every refusal removes the verdict before it returns, so an
349// error here never leaves an old verdict standing.
350func judgePrevWith(optDir, dataDir string, freeze holdFreezer, learn func() string) error {
351 deny := func(cause error) error {
352 if err := removeDurable(prevAllowedPath(optDir)); err != nil {
353 return errors.Join(cause, fmt.Errorf("remove the verdict on carlos.prev: %w", err))
354 }
355 return cause
356 }
357 if _, err := os.Stat(systemPrevPath(optDir)); err != nil {
358 return deny(nil)
359 }
360 if _, ok := readLine(prevVersionPath(optDir)); !ok && learn != nil {
361 if f, _ := readLocalFloor(dataDir); f != nil {
362 if v := learn(); validSystemVersion(v) {
363 if err := writePrevVersion(optDir, v); err != nil {
364 return deny(err)
365 }
366 }
367 }
368 }
369 ran := false
370 err := withFloorLock(dataDir, false, func(*movesDir) error {
371 ran = true
372 floor, err := readLocalFloor(dataDir)
373 if err != nil {
374 return deny(fmt.Errorf("moves protocol floor: the floor on this box cannot be read: %v", err))
375 }
376 if floor == nil {
377 return deny(nil)
378 }
379 version, ok := readLine(prevVersionPath(optDir))
380 if !ok {
381 return deny(nil)
382 }
383 if at, _ := moves.ReleaseAtOrAbove(version, floor.MinRelease); !at {
384 return deny(nil)
385 }
386 called := false
387 err = freeze(func(held []registry.Route) error {
388 called = true
389 if len(held) > 0 {
390 return deny(nil)
391 }
392 return writeFileDurable(prevAllowedPath(optDir), []byte(floor.MinRelease+"\n"), 0o644)
393 })
394 if !called {
395 return deny(fmt.Errorf("held rows: %w", err))
396 }
397 return err
398 })
399 if !ran {
400 return deny(fmt.Errorf("moves protocol floor: the floor on this box cannot be read: %v", err))
401 }
402 return err
403}
404
405// prevLockFile serialises the two writers of carlos.prev's sidecar and
406// verdict: decidePrev, which replaces carlos.prev under the apply lock, and
407// the root adopt pass, which re-judges it under no apply lock at all.
408// Without it an adopt pass that read the old sidecar could write its
409// verdict after decidePrev had replaced carlos.prev and removed it.
410const prevLockFile = "prev.lock"
411
412const (
413 prevLockWait = 2 * time.Minute
414 prevLockPoll = 100 * time.Millisecond
415)
416
417// withPrevLock runs fn holding prevLockFile, waiting up to prevLockWait.
418// Holders are short (a binary copy at most), so a lock that is not free by
419// then is reported, not waited out.
420func withPrevLock(optDir string, fn func() error) error {
421 dir := systemRootDir(optDir)
422 if err := os.MkdirAll(dir, 0o755); err != nil {
423 return err
424 }
425 lock, err := os.OpenFile(filepath.Join(dir, prevLockFile), os.O_CREATE|os.O_RDWR, 0o600)
426 if err != nil {
427 return err
428 }
429 defer lock.Close()
430 deadline := time.Now().Add(prevLockWait)
431 for {
432 err := flockExclusiveNonblock(lock)
433 if err == nil {
434 break
435 }
436 if errors.Is(err, errSystemUnsupported) || time.Now().After(deadline) {
437 return fmt.Errorf("carlos.prev lock: %w", err)
438 }
439 time.Sleep(prevLockPoll)
440 }
441 return fn()
442}
443
444// rootJudgePrev is the root adopt pass's judgement (every mode): the moves
445// half, then the seeds half (seedsfloor.go). Each runs whatever the other
446// returns.
447func rootJudgePrev(reg *registry.Registry) error {
448 return rootJudgePrevWithSource(reg, rootMovesFloorIdentity)
449}
450
451func rootJudgePrevWithSource(reg *registry.Registry, source rootFloorIdentitySource) error {
452 if id, required, err := source.read(); err != nil {
453 return err
454 } else if required {
455 return errors.Join(rootJudgeMovesPrevWithSource(reg, source), rootRefreshAndJudgeSeedsPrev(id))
456 }
457 return errors.Join(rootJudgeMovesPrevWithSource(reg, source), rootJudgeSeedsPrev())
458}
459
460func rootJudgeMovesPrevWithSource(reg *registry.Registry, source rootFloorIdentitySource) error {
461 if id, required, err := source.read(); err != nil {
462 return err
463 } else if required {
464 return rootRefreshAndJudgeMovesPrev(id)
465 }
466
467 optDir, dd := release.OptDir(), dataDir()
468 if _, err := os.Stat(filepath.Join(dd, moves.FloorFile)); errors.Is(err, os.ErrNotExist) {
469 // No floor armed: nothing to judge and no verdict to keep. The one
470 // stat is all an unarmed box pays.
471 if _, err := os.Stat(prevAllowedPath(optDir)); errors.Is(err, os.ErrNotExist) {
472 return nil
473 }
474 }
475 return withPrevLock(optDir, func() error {
476 return judgePrevWith(optDir, dd, reg.WithHoldsFrozen, func() string {
477 out, err := productionSysOps().run(context.Background(), systemVersionWait, systemPrevPath(optDir), "version")
478 if err != nil {
479 return ""
480 }
481 return carlosVersionOf(out)
482 })
483 })
484}
485
486// movesFloorRefresh is one pass of the agent's refresh: syncLocalFloor on
487// the agent's own store handle, logged when it fails, then the seeds
488// floor's (seedsFloorRefresh) on the same handle.
489func movesFloorRefresh(ctx context.Context, s release.Store, dataDir string, logf func(string, ...any)) {
490 if _, err := syncLocalFloor(ctx, s, dataDir); err != nil {
491 logf("agent: moves protocol floor refresh: %v", err)
492 }
493 seedsFloorRefresh(ctx, s, dataDir, logf)
494}
495
496// movesFloorLoop refreshes the local floor files at start and every hour.
497// Started by edgesFileLoop, whose file publishes what it writes.
498func movesFloorLoop(ctx context.Context, s release.Store, dataDir string) {
499 movesFloorRefresh(ctx, s, dataDir, log.Printf)
500 tick := time.NewTicker(movesFloorRefreshEvery)
501 defer tick.Stop()
502 for range tick.C {
503 movesFloorRefresh(ctx, s, dataDir, log.Printf)
504 }
505}
506
507// withMovesFields sets the edges file's four moves fields from this box's
508// own files, read at each write: the protocol this build speaks, the armed
509// floor's min_release (absent: not armed), carlos.prev's release ("none":
510// there is no carlos.prev; absent: unknown) and the verdict (absent: no
511// verdict, which the door reads as below the floor).
512func withMovesFields(f edgesFile, dataDir, optDir string) edgesFile {
513 // The public root marker is the agent's mode indication; it cannot inspect
514 // the private census. G2 installs this marker before completing migration.
515 if _, err := os.Lstat(rootFloorDir(optDir)); !errors.Is(err, os.ErrNotExist) {
516 return withRootMovesFields(f, optDir)
517 }
518
519 f.MovesProtocol = moves.ProtocolVersion
520 // From floor.json, the file every Go reader trusts: floor.min-release
521 // is written first and can be ahead of it (writeFloorFiles).
522 f.MovesFloorMinRelease = ""
523 if fl, err := readLocalFloor(dataDir); err == nil && fl != nil {
524 f.MovesFloorMinRelease = fl.MinRelease
525 }
526 f.PrevRelease = ""
527 if _, err := os.Stat(systemPrevPath(optDir)); errors.Is(err, os.ErrNotExist) {
528 f.PrevRelease = "none"
529 } else if err == nil {
530 f.PrevRelease, _ = readLine(prevVersionPath(optDir))
531 }
532 f.PrevAllowedFor, _ = readLine(prevAllowedPath(optDir))
533 return f
534}
535