amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "bytes"
5 "context"
6 "encoding/json"
7 "errors"
8 "os"
9 "path/filepath"
10 "reflect"
11 "slices"
12 "sort"
13 "strings"
14 "sync"
15 "testing"
16 "time"
17
18 "github.com/carlosframework/platform/internal/activator"
19 "github.com/carlosframework/platform/internal/cgroup"
20 "github.com/carlosframework/platform/internal/domains"
21 "github.com/carlosframework/platform/internal/hostwrite"
22 "github.com/carlosframework/platform/internal/instancectl"
23 "github.com/carlosframework/platform/internal/layout"
24 "github.com/carlosframework/platform/internal/moves"
25 "github.com/carlosframework/platform/internal/provision"
26 "github.com/carlosframework/platform/internal/registry"
27 "github.com/carlosframework/platform/internal/release"
28)
29
30// The offline operator move's box side (Task 3 of
31// docs/superpowers/plans/2026-10-20-offline-operator-move.md), tested
32// through the functions the agent calls: newMovesWorker, armOffline,
33// reconcileSerializedWith, the production edges-file writer and movesPass,
34// over one DirStore, real registries and real activator services (the
35// package's fake wake backend, which only the fence needs).
36
37const (
38 offHost = "shop.example.com"
39 offID = "mv_zyxwvutsrqponmlkjihgfedcba"
40 offID2 = "mv_abcdefghijklmnopqrstuvwxy2"
41)
42
43type offWorld struct {
44 t *testing.T
45 s release.Store
46 acct string
47 mu sync.Mutex
48 now time.Time
49}
50
51func (wd *offWorld) clock() time.Time {
52 wd.mu.Lock()
53 defer wd.mu.Unlock()
54 return wd.now
55}
56
57func (wd *offWorld) advance(d time.Duration) {
58 wd.mu.Lock()
59 wd.now = wd.now.Add(d)
60 wd.mu.Unlock()
61}
62
63// newOffWorld is a deployment with one account, one app and one instance
64// record for offHost at the home placement ("").
65func newOffWorld(t *testing.T) *offWorld {
66 t.Helper()
67 t.Setenv("CARLOS_HOST_CLAIM", "") // the appconfig and provision guards read the process env
68 ctx := context.Background()
69 wd := &offWorld{t: t, s: release.NewDirStore(t.TempDir()), acct: domains.Sqid(2), now: time.Date(2026, 10, 20, 9, 0, 0, 0, time.UTC)}
70 manifest, err := json.Marshal(map[string]any{"id": 2, "sqid": wd.acct, "name": "fixture", "created_at": wd.now})
71 if err != nil {
72 t.Fatal(err)
73 }
74 if _, err := wd.s.Put(ctx, "accounts/"+wd.acct+"/account.json", manifest, ""); err != nil {
75 t.Fatal(err)
76 }
77 if err := provision.PutManifest(ctx, wd.s, wd.acct, "app", provision.Manifest{Schema: 1, App: "app", Domain: "example.com"}); err != nil {
78 t.Fatal(err)
79 }
80 wd.create(offHost)
81 return wd
82}
83
84func (wd *offWorld) create(host string) {
85 wd.t.Helper()
86 rec := provision.Record{Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], Account: wd.acct, App: "app", Region: "", Channel: "stable", CreatedAt: wd.now}
87 if err := provision.CreateRecord(context.Background(), wd.s, rec); err != nil {
88 wd.t.Fatal(err)
89 }
90}
91
92// move applies one offline transition to offHost's record.
93func (wd *offWorld) move(mut func(*provision.Record) error) provision.Record {
94 wd.t.Helper()
95 return wd.moveHost(offHost, mut)
96}
97
98func (wd *offWorld) moveHost(host string, mut func(*provision.Record) error) provision.Record {
99 wd.t.Helper()
100 rec, err := provision.SetRecordMove(context.Background(), wd.s, wd.acct, "app", host, mut)
101 if err != nil {
102 wd.t.Fatal(err)
103 }
104 return *rec
105}
106
107func (wd *offWorld) record() provision.Record {
108 wd.t.Helper()
109 rec, err := provision.GetRecord(context.Background(), wd.s, wd.acct, "app", offHost)
110 if err != nil {
111 wd.t.Fatal(err)
112 }
113 return *rec
114}
115
116// The legal path to a state, from the record at rest.
117func (wd *offWorld) start() {
118 wd.move(provision.OfflineStart(offID, "", "uk", "op@example.com", wd.clock()))
119}
120func (wd *offWorld) seal() { wd.move(provision.OfflineSeal(offID, "ie-a", []string{"ie-a", "ie-b"})) }
121func (wd *offWorld) sealed() {
122 wd.move(provision.OfflineSealedBy(offID, "ie-a", provision.MoveSource{}))
123}
124func (wd *offWorld) flip() { wd.move(provision.OfflineFlip(offID)) }
125func (wd *offWorld) landed() { wd.move(provision.OfflineLandedBy(offID, "uk")) }
126func (wd *offWorld) open() { wd.move(provision.OfflineOpenTransition(offID, wd.clock())) }
127func (wd *offWorld) finishing() {
128 wd.move(provision.OfflineFinishingTransition(offID))
129}
130func (wd *offWorld) abort() { wd.move(provision.OfflineAbort(offID)) }
131
132// report reads one box's report for offID, or fails.
133func (wd *offWorld) report(role, box string) moves.OfflineReport {
134 wd.t.Helper()
135 r, ok := wd.reportIf(role, box)
136 if !ok {
137 wd.t.Fatalf("no %s report from %s", role, box)
138 }
139 return r
140}
141
142func (wd *offWorld) reportIf(role, box string) (moves.OfflineReport, bool) {
143 wd.t.Helper()
144 body, _, err := wd.s.Get(context.Background(), moves.OfflineReportKey(offHost, offID, role, box))
145 if errors.Is(err, os.ErrNotExist) {
146 return moves.OfflineReport{}, false
147 }
148 if err != nil {
149 wd.t.Fatal(err)
150 }
151 var r moves.OfflineReport
152 if err := json.Unmarshal(body, &r); err != nil {
153 wd.t.Fatal(err)
154 }
155 if err := moves.ValidateOfflineReport(r); err != nil {
156 wd.t.Fatal(err)
157 }
158 return r, true
159}
160
161// offBox is one box: its own data directory, registry, activator and moves
162// worker, armed as armMovesPass arms it (armOffline).
163type offBox struct {
164 wd *offWorld
165 name, region string
166 dd, opt, cg string
167 env map[string]string
168 reg *registry.Registry
169 svc *activator.Service
170 w *movesWorker
171 nested bool
172 last []byte
173}
174
175func (wd *offWorld) box(name, region string, env map[string]string) *offBox {
176 wd.t.Helper()
177 b := &offBox{wd: wd, name: name, region: region, dd: wd.t.TempDir(), opt: wd.t.TempDir(), cg: wd.t.TempDir(), env: env}
178 reg, err := registry.Open(filepath.Join(b.dd, "registry.db"))
179 if err != nil {
180 wd.t.Fatal(err)
181 }
182 wd.t.Cleanup(func() { reg.Close() })
183 b.reg = reg
184 b.restart()
185 return b
186}
187
188func (b *offBox) getenv(k string) string {
189 if v, ok := b.env[k]; ok {
190 return v
191 }
192 if k == "CARLOS_HIBERNATE_BUCKET" {
193 return "data"
194 }
195 return ""
196}
197
198// restart is an agent restart: a fresh activator (no fences) and a fresh
199// worker over the same registry and data directory.
200func (b *offBox) restart() {
201 b.svc = activator.New(activator.Config{Bucket: "data", Idle: time.Hour}, &fakeWakeBackend{}, b.reg)
202 cfg := moveCfg{Region: b.region, Box: b.name, DataDir: b.dd, OptDir: b.opt, Quiesced: b.svc.Quiesced, Route: b.reg.Get, Now: b.wd.clock}
203 armOffline(&cfg, b.dd, b.reg, b.svc, nil, hostwrite.Config{}, b.getenv)
204 // The stage-requested marker is root's business, and its throttled
205 // timer would outlive the test's directories.
206 cfg.Offline = newOfflineView(cfg, nil)
207 cfg.ReleaseHold = releaseHeld(b.reg, nil)
208 cfg.Procs = func(host string) (int, error) { return cgroup.HostProcs(b.cg, host, "") }
209 b.w = newMovesWorker(cfg)
210}
211
212func (b *offBox) reconcile() {
213 b.wd.t.Helper()
214 cfg := provision.ReconcileConfig{Region: b.region, DataDir: b.dd}
215 if b.nested {
216 cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }
217 } else {
218 cfg.Layout = func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }
219 }
220 if err := reconcileSerializedWith(context.Background(), b.wd.s, b.reg, cfg, b.w); err != nil {
221 b.wd.t.Fatal(err)
222 }
223}
224
225func (b *offBox) edges() {
226 b.wd.t.Helper()
227 if err := edgesFileWrite(context.Background(), b.wd.s, b.reg, edgesFile{Box: b.name, Placement: b.region}, "", &b.last, false); err != nil {
228 b.wd.t.Fatal(err)
229 }
230}
231
232func (b *offBox) pass() {
233 b.wd.t.Helper()
234 if err := movesPass(context.Background(), b.wd.s, b.w); err != nil {
235 b.wd.t.Fatal(err)
236 }
237}
238
239// tick is the agent's order on a serial bump: reconcile, the edges file,
240// the moves pass.
241func (b *offBox) tick() {
242 b.wd.t.Helper()
243 b.reconcile()
244 b.edges()
245 b.pass()
246}
247
248func (b *offBox) row(host string) *registry.Route {
249 b.wd.t.Helper()
250 rt, err := b.reg.Get(host)
251 if err != nil {
252 b.wd.t.Fatal(err)
253 }
254 return rt
255}
256
257// lists reports whether this box's published edges file lists host.
258func (b *offBox) lists(host string) bool {
259 b.wd.t.Helper()
260 body, _, err := b.wd.s.Get(context.Background(), edgesFileKey(b.name))
261 if err != nil {
262 b.wd.t.Fatal(err)
263 }
264 var f edgesFile
265 if err := json.Unmarshal(body, &f); err != nil {
266 b.wd.t.Fatal(err)
267 }
268 _, ok := f.Hosts[host]
269 return ok
270}
271
272func writeFile(t *testing.T, p, body string) {
273 t.Helper()
274 if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
275 t.Fatal(err)
276 }
277 if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
278 t.Fatal(err)
279 }
280}
281
282// --- the hold ---------------------------------------------------------------
283
284// offlineHold's table: a pure function of the record and the box's
285// placement. The home's "" and a named region; every state.
286func TestOfflineHoldIsAFunctionOfTheRecord(t *testing.T) {
287 home := moveCfg{Region: ""}
288 uk := moveCfg{Region: "uk"}
289 fr := moveCfg{Region: "fr"}
290 rec := func(region, state string) provision.Record {
291 return provision.Record{Host: offHost, Region: region, Move: &provision.MoveState{Kind: provision.MoveKindOffline, ID: offID, From: "", To: "uk", State: state}}
292 }
293 cases := []struct {
294 name string
295 box moveCfg
296 rec provision.Record
297 wantStatus, wantOp string
298 }{
299 {"source stopping", home, rec("", provision.OfflineStopping), "moving", offID},
300 {"source sealing", home, rec("", provision.OfflineSealing), "moving", offID},
301 {"source sealed", home, rec("", provision.OfflineSealed), "moving", offID},
302 {"source reverting", home, rec("", provision.OfflineReverting), "", ""},
303 {"source after the flip", home, rec("uk", provision.OfflineLanding), "", ""},
304 {"target before the flip", uk, rec("", provision.OfflineSealed), "", ""},
305 {"target landing", uk, rec("uk", provision.OfflineLanding), "landing", offID},
306 {"target landed", uk, rec("uk", provision.OfflineLanded), "landing", offID},
307 {"target landing, region not yet the target's", uk, rec("", provision.OfflineLanding), "", ""},
308 {"target open", uk, rec("uk", provision.OfflineOpen), "", ""},
309 {"target finishing", uk, rec("uk", provision.OfflineFinishing), "", ""},
310 {"target reverting", uk, rec("", provision.OfflineReverting), "", ""},
311 {"a third region", fr, rec("", provision.OfflineStopping), "", ""},
312 {"a third region after the flip", fr, rec("uk", provision.OfflineLanding), "", ""},
313 }
314 for _, c := range cases {
315 t.Run(c.name, func(t *testing.T) {
316 status, op, ok := offlineHold(c.box)(c.rec)
317 if status != c.wantStatus || op != c.wantOp || ok != (c.wantStatus != "") {
318 t.Fatalf("hold = (%q, %q, %v), want (%q, %q)", status, op, ok, c.wantStatus, c.wantOp)
319 }
320 })
321 }
322 // A record at rest, a legacy move and a Phase 1 move are never held.
323 for _, m := range []*provision.MoveState{nil, {To: "uk", State: provision.MoveCutting}, {ID: offID, From: "", To: "uk", State: provision.OfflineStopping}} {
324 if _, _, ok := offlineHold(home)(provision.Record{Host: offHost, Move: m}); ok {
325 t.Fatalf("held: %+v", m)
326 }
327 }
328}
329
330// B1, the pinning test: a fresh target worker whose keep view has never
331// seen the move (built from a list with no offline record in it, as a
332// box's view is before the flip) mints the row held "landing" on its first
333// reconcile after the flip. And with the view never built at all, reconcile
334// through the offline hooks alone does the same.
335func TestOfflineFreshTargetMintsTheLandingRowHeld(t *testing.T) {
336 wd := newOffWorld(t)
337 wd.start()
338 wd.seal()
339 wd.sealed()
340 wd.flip()
341
342 uk := wd.box("uk-box", "uk", nil)
343 uk.w.cfg.Offline.refresh(nil, nil) // stale: no offline move in it
344 uk.reconcile()
345 rt := uk.row(offHost)
346 if rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
347 t.Fatalf("after the flip the target minted %+v, want held landing by %s", rt, offID)
348 }
349 uk.edges()
350 if uk.lists(offHost) {
351 t.Fatal("the target's edges file lists a held landing row")
352 }
353
354 // The hooks with a view that was never built, straight into
355 // provision.Reconcile: the hold needs no view.
356 fresh := wd.box("uk-box-2", "uk", nil)
357 if fresh.w.cfg.Offline.built() {
358 t.Fatal("fixture: the view is built")
359 }
360 cfg := withOfflineHolds(provision.ReconcileConfig{Region: "uk", DataDir: fresh.dd}, offlineHold(fresh.w.cfg), fresh.w.cfg.Offline)
361 if err := provision.Reconcile(context.Background(), wd.s, fresh.reg, cfg); err != nil {
362 t.Fatal(err)
363 }
364 if rt := fresh.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
365 t.Fatalf("an unbuilt view: minted %+v", rt)
366 }
367}
368
369// --- the source boxes -------------------------------------------------------
370
371// start holds the row on both source boxes (one minting it held, one
372// holding the row it has, keeping its other columns), both report held,
373// fenced and quiesced, and neither edges file lists the host. An agent
374// restart re-fences and re-reports. After the flip the rows stay, kept.
375func TestOfflineStartHoldsEverySourceRow(t *testing.T) {
376 wd := newOffWorld(t)
377 a := wd.box("ie-a", "", nil)
378 b := wd.box("ie-b", "", nil)
379 uk := wd.box("uk-box", "uk", nil)
380 a.tick() // ie-a serves the host before the move; ie-b has no row yet
381 if rt := a.row(offHost); rt == nil || rt.Status != "ready" || !a.lists(offHost) {
382 t.Fatalf("fixture: ie-a does not serve the host: %+v", rt)
383 }
384 if err := a.reg.SetReplication(offHost, "daily"); err != nil {
385 t.Fatal(err)
386 }
387 if err := a.reg.SetNextDue(offHost, 1234); err != nil {
388 t.Fatal(err)
389 }
390 before := *a.row(offHost)
391
392 wd.start()
393 a.tick()
394 b.tick()
395 uk.tick()
396 for _, bx := range []*offBox{a, b} {
397 rt := bx.row(offHost)
398 if rt == nil || rt.Status != "moving" || rt.HoldOp != offID {
399 t.Fatalf("%s: row %+v, want held moving by the move", bx.name, rt)
400 }
401 if bx.lists(offHost) {
402 t.Fatalf("%s's edges file lists a held host", bx.name)
403 }
404 if !bx.svc.Fenced(offHost) {
405 t.Fatalf("%s: no wake fence", bx.name)
406 }
407 r := wd.report(moves.OfflineRoleSource, bx.name)
408 if !r.Held || !r.Fenced || !r.Quiesced || !r.RowPresent || r.Procs != 0 || r.LayoutJournal || r.Phase != moves.OfflinePhaseStopped || !r.At.Equal(wd.clock()) {
409 t.Fatalf("%s: report %+v", bx.name, r)
410 }
411 }
412 after := *a.row(offHost)
413 after.Status, after.HoldOp = before.Status, before.HoldOp
414 if !reflect.DeepEqual(after, before) {
415 t.Fatalf("holding the existing row changed its other columns:\n before %+v\n after %+v", before, after)
416 }
417 if uk.row(offHost) != nil {
418 t.Fatal("the target minted a row before the flip")
419 }
420 if _, ok := wd.reportIf(moves.OfflineRoleSource, "uk-box"); ok {
421 t.Fatal("the target wrote a source report")
422 }
423
424 // An agent restart: the fence is gone with the process, and the next
425 // pass installs it again and rewrites the report.
426 a.restart()
427 if a.svc.Fenced(offHost) {
428 t.Fatal("fixture: a fresh activator is fenced")
429 }
430 wd.advance(time.Minute)
431 a.tick()
432 if !a.svc.Fenced(offHost) {
433 t.Fatal("after a restart the pass did not fence again")
434 }
435 if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.Fenced || !r.At.Equal(wd.clock()) {
436 t.Fatalf("after a restart: %+v", r)
437 }
438
439 // The flip: the record names uk; both source rows stay, held, and keep
440 // reporting; the target mints its row held landing.
441 wd.seal()
442 wd.sealed()
443 wd.flip()
444 wd.advance(time.Minute)
445 for _, bx := range []*offBox{a, b, uk} {
446 bx.tick()
447 }
448 for _, bx := range []*offBox{a, b} {
449 if rt := bx.row(offHost); rt == nil || rt.Status != "moving" || rt.HoldOp != offID {
450 t.Fatalf("%s after the flip: %+v, want the row kept, held", bx.name, rt)
451 }
452 if r := wd.report(moves.OfflineRoleSource, bx.name); !r.Held || !r.Fenced || !r.At.Equal(wd.clock()) {
453 t.Fatalf("%s after the flip: %+v", bx.name, r)
454 }
455 }
456 if rt := uk.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
457 t.Fatalf("target after the flip: %+v", rt)
458 }
459 if uk.lists(offHost) {
460 t.Fatal("the target lists the landing row")
461 }
462}
463
464// What the report reads from disk: a box with only a meta dir has local
465// state; a process left in the host's cgroup and a layout journal show.
466func TestOfflineSourceReportReadsTheBox(t *testing.T) {
467 wd := newOffWorld(t)
468 a := wd.box("ie-a", "", nil)
469 b := wd.box("ie-b", "", nil)
470 a.tick()
471 wd.start()
472 // ie-b has no DB file, only the activator's meta dir with one LTX file.
473 meta := activatorMetaDir(provision.DefaultDBPath(b.dd, offHost))
474 writeFile(t, filepath.Join(meta, "ltx", "0", "0000000000000001-0000000000000007.ltx"), "x")
475 // ie-a: files, env, a process in the host's exec cgroup, a journal.
476 writeFile(t, filepath.Join(a.dd, offHost, "instance.key"), "k")
477 writeFile(t, filepath.Join(a.dd, offHost+".env"), "CARLOS_INSTANCE_KEY=k\n")
478 writeFile(t, filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs"), "4242\n")
479 writeFile(t, provision.LayoutJournalPath(a.dd, offHost), "{}")
480 a.tick()
481 b.tick()
482 rb := wd.report(moves.OfflineRoleSource, "ie-b")
483 if !rb.LocalDB || rb.LocalTXID != 7 || rb.Files || rb.Env || rb.Sidecar {
484 t.Fatalf("a meta dir alone: %+v", rb)
485 }
486 ra := wd.report(moves.OfflineRoleSource, "ie-a")
487 if ra.LocalDB || !ra.Files || !ra.Env || ra.Procs != 1 || !ra.LayoutJournal {
488 t.Fatalf("ie-a: %+v", ra)
489 }
490 // A box that cannot count its processes reports -1, never 0.
491 a.w.cfg.Procs = func(string) (int, error) { return 0, cgroup.ErrUnknown }
492 a.pass()
493 if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Procs != -1 {
494 t.Fatalf("unknown processes reported as %d", r.Procs)
495 }
496 // A sidecar file shows.
497 writeFile(t, filepath.Join(b.dd, offHost+"-sidecar.env"), "X=1\n")
498 b.pass()
499 if r := wd.report(moves.OfflineRoleSource, "ie-b"); !r.Sidecar {
500 t.Fatalf("sidecar not seen: %+v", r)
501 }
502}
503
504// offline-reverting, before and after the flip: every source row is
505// released and served again, the fence is lifted, the boxes report
506// reverted, and the next reconcile does not hold the rows again.
507func TestOfflineRevertReleasesEverySourceRow(t *testing.T) {
508 for _, afterFlip := range []bool{false, true} {
509 t.Run(map[bool]string{false: "before the flip", true: "after the flip"}[afterFlip], func(t *testing.T) {
510 wd := newOffWorld(t)
511 a := wd.box("ie-a", "", nil)
512 b := wd.box("ie-b", "", nil)
513 uk := wd.box("uk-box", "uk", nil)
514 a.tick()
515 b.tick()
516 wd.start()
517 if afterFlip {
518 wd.seal()
519 wd.sealed()
520 wd.flip()
521 }
522 for _, bx := range []*offBox{a, b, uk} {
523 bx.tick()
524 }
525 wd.abort()
526 for i := 0; i < 2; i++ {
527 for _, bx := range []*offBox{a, b, uk} {
528 bx.tick()
529 }
530 }
531 for _, bx := range []*offBox{a, b} {
532 rt := bx.row(offHost)
533 if rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
534 t.Fatalf("%s: %+v, want ready and unheld", bx.name, rt)
535 }
536 if !bx.lists(offHost) {
537 t.Fatalf("%s does not serve the host again", bx.name)
538 }
539 if bx.svc.Fenced(offHost) {
540 t.Fatalf("%s: the fence is still up", bx.name)
541 }
542 if r := wd.report(moves.OfflineRoleSource, bx.name); r.Phase != moves.OfflinePhaseReverted || r.Held {
543 t.Fatalf("%s: %+v", bx.name, r)
544 }
545 }
546 // The move finishes; nothing holds the rows again.
547 wd.move(provision.OfflineFinish(offID, wd.clock()))
548 for _, bx := range []*offBox{a, b} {
549 bx.tick()
550 if rt := bx.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
551 t.Fatalf("%s after finish: %+v", bx.name, rt)
552 }
553 }
554 })
555 }
556}
557
558// offline-open on a flat source: every path of the source-retire row that
559// is there is renamed into the park directory byte for byte, nothing is
560// deleted, the fence is lifted, and the parked report lists every path with
561// its action. A process left in the host's cgroup stops the park first.
562// offline-finishing then removes the row and the box reports gone.
563func TestOfflineOpenParksTheFlatSource(t *testing.T) {
564 wd := newOffWorld(t)
565 a := wd.box("ie-a", "", nil)
566 uk := wd.box("uk-box", "uk", nil)
567 a.tick()
568 db := a.row(offHost).DBPath
569 contents := map[string]string{
570 db: "sqlite",
571 db + "-wal": "wal",
572 filepath.Join(activatorMetaDir(db), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx",
573 filepath.Join(a.dd, offHost, "instance.key"): "key",
574 filepath.Join(a.dd, offHost+".env"): "CARLOS_INSTANCE_KEY=k\n",
575 filepath.Join(a.dd, offHost+".config.env"): "SECRET=s\n",
576 filepath.Join(a.dd, "programs", offHost+".json"): `{"programs":[]}`,
577 filepath.Join(a.opt, "live", offHost, "app"): "binary",
578 filepath.Join(a.dd, "neighbour.example.com.db"): "not ours",
579 }
580 for p, body := range contents {
581 writeFile(t, p, body)
582 }
583 wd.start()
584 a.tick() // the source reports before the seal, as the seal door requires
585 wd.seal()
586 wd.sealed()
587 wd.flip()
588 wd.landed()
589 a.tick()
590 uk.tick()
591 wd.open()
592
593 // A process still in the host's cgroup: nothing is renamed.
594 procs := filepath.Join(a.cg, cgroup.ExecName(offHost), "cgroup.procs")
595 writeFile(t, procs, "4242\n")
596 a.tick()
597 if _, err := os.Stat(db); err != nil {
598 t.Fatalf("parked with a process left: %v", err)
599 }
600 if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); !ok {
601 t.Fatal("fixture: no report")
602 }
603 if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase == moves.OfflinePhaseParked {
604 t.Fatalf("reported parked with a process left: %+v", r)
605 }
606 writeFile(t, procs, "")
607
608 a.tick()
609 park := moves.OfflineParkDir(a.dd, offHost, offID, moves.OfflineRoleSource)
610 for p, body := range contents {
611 switch {
612 case strings.HasPrefix(p, a.opt), strings.Contains(p, "neighbour"):
613 if got, err := os.ReadFile(p); err != nil || string(got) != body {
614 t.Fatalf("%s was touched: %q %v", p, got, err)
615 }
616 continue
617 }
618 if _, err := os.Lstat(p); !errors.Is(err, os.ErrNotExist) {
619 t.Fatalf("%s is still in place: %v", p, err)
620 }
621 }
622 for rel, body := range map[string]string{
623 filepath.Base(db): "sqlite",
624 filepath.Base(db) + "-wal": "wal",
625 filepath.Join(filepath.Base(activatorMetaDir(db)), "ltx", "0", "0000000000000001-0000000000000003.ltx"): "ltx",
626 filepath.Join(offHost, "instance.key"): "key",
627 offHost + ".env": "CARLOS_INSTANCE_KEY=k\n",
628 offHost + ".config.env": "SECRET=s\n",
629 offHost + ".json": `{"programs":[]}`,
630 } {
631 if got, err := os.ReadFile(filepath.Join(park, rel)); err != nil || string(got) != body {
632 t.Fatalf("park/%s = %q %v, want %q", rel, got, err, body)
633 }
634 }
635 r := wd.report(moves.OfflineRoleSource, "ie-a")
636 if r.Phase != moves.OfflinePhaseParked || r.LocalDB || r.Files || r.Env || r.Fenced {
637 t.Fatalf("parked report: %+v", r)
638 }
639 if a.svc.Fenced(offHost) {
640 t.Fatal("the fence is still up after the park")
641 }
642 want := map[string]string{
643 db: moves.RetiredParked, db + "-wal": moves.RetiredParked, db + "-shm": moves.RetiredAbsent,
644 db + "-litestream": moves.RetiredAbsent, activatorMetaDir(db): moves.RetiredParked,
645 filepath.Join(a.dd, offHost): moves.RetiredParked, filepath.Join(a.dd, offHost+".env"): moves.RetiredParked,
646 filepath.Join(a.dd, offHost+".config.env"): moves.RetiredParked, filepath.Join(a.dd, "programs", offHost+".json"): moves.RetiredParked,
647 filepath.Join(a.opt, "live", offHost): moves.RetiredLeft,
648 }
649 assertRetired(t, r.Retired, want)
650
651 // A second pass changes nothing and reports the same inventory.
652 before := r.Retired
653 a.tick()
654 if r := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(r.Retired, before) || r.Phase != moves.OfflinePhaseParked {
655 t.Fatalf("second pass: %+v", r.Retired)
656 }
657 // The row stays held through open.
658 if rt := a.row(offHost); rt == nil || rt.HoldOp != offID {
659 t.Fatalf("row at open: %+v", rt)
660 }
661
662 // finishing: keep no longer holds the row; reconcile removes it and
663 // the box reports gone. The parked state stays.
664 // The view the reconcile reads is the last moves pass's, so the row
665 // goes on the pass after the one that first sees finishing.
666 wd.finishing()
667 a.tick()
668 a.tick()
669 if rt := a.row(offHost); rt != nil {
670 t.Fatalf("row at finishing: %+v", rt)
671 }
672 a.pass()
673 if r := wd.report(moves.OfflineRoleSource, "ie-a"); r.Phase != moves.OfflinePhaseGone || r.RowPresent || r.LocalDB {
674 t.Fatalf("gone report: %+v", r)
675 }
676 if got, err := os.ReadFile(filepath.Join(park, filepath.Base(db))); err != nil || string(got) != "sqlite" {
677 t.Fatalf("the parked DB: %q %v", got, err)
678 }
679}
680
681func assertRetired(t *testing.T, got []moves.RetiredPath, want map[string]string) {
682 t.Helper()
683 seen := map[string]string{}
684 for _, rp := range got {
685 if _, dup := seen[rp.Path]; dup {
686 t.Fatalf("%s listed twice", rp.Path)
687 }
688 seen[rp.Path] = rp.Action
689 if rp.Action == moves.RetiredLeft && rp.Reason == "" {
690 t.Fatalf("%s left with no reason", rp.Path)
691 }
692 }
693 for p, action := range want {
694 if seen[p] != action {
695 t.Fatalf("%s: %q, want %q (inventory %+v)", p, seen[p], action, got)
696 }
697 }
698 if len(seen) != len(want) {
699 var extra []string
700 for p := range seen {
701 if _, ok := want[p]; !ok {
702 extra = append(extra, p+"="+seen[p])
703 }
704 }
705 sort.Strings(extra)
706 t.Fatalf("unexpected entries: %v", extra)
707 }
708}
709
710// M3: a path the agent can never rename (here a file in a directory it may
711// not write, as a root-owned /data/programs is) is reported left once, with
712// its reason, and is not tried again: later passes, an agent restart
713// included, neither fence nor rename, and the report stays parked.
714func TestOfflineParkLeavesWhatCannotBeRenamedOnce(t *testing.T) {
715 if os.Geteuid() == 0 {
716 t.Skip("root can rename anything")
717 }
718 wd := newOffWorld(t)
719 a := wd.box("ie-a", "", nil)
720 a.tick()
721 db := a.row(offHost).DBPath
722 writeFile(t, db, "sqlite")
723 programs := filepath.Join(a.dd, "programs")
724 writeFile(t, filepath.Join(programs, offHost+".json"), "{}")
725 if err := os.Chmod(programs, 0o555); err != nil {
726 t.Fatal(err)
727 }
728 t.Cleanup(func() { os.Chmod(programs, 0o755) })
729 fences := 0
730 count := func() {
731 inner := a.w.cfg.FenceAs
732 a.w.cfg.FenceAs = func(ctx context.Context, host string, gen uint64, owner string) error {
733 fences++
734 return inner(ctx, host, gen, owner)
735 }
736 }
737 wd.start()
738 a.tick()
739 wd.seal()
740 wd.sealed()
741 wd.flip()
742 wd.landed()
743 a.tick()
744 wd.open()
745 count()
746 before := fences
747 a.tick()
748 if fences != before+1 {
749 t.Fatalf("the first park fenced %d times", fences-before)
750 }
751 r := wd.report(moves.OfflineRoleSource, "ie-a")
752 if r.Phase != moves.OfflinePhaseParked {
753 t.Fatalf("phase %s", r.Phase)
754 }
755 var left moves.RetiredPath
756 for _, rp := range r.Retired {
757 if rp.Path == filepath.Join(programs, offHost+".json") {
758 left = rp
759 }
760 if rp.Path == db && rp.Action != moves.RetiredParked {
761 t.Fatalf("the DB: %+v", rp)
762 }
763 }
764 if left.Action != moves.RetiredLeft || !strings.HasPrefix(left.Reason, leftCannot) {
765 t.Fatalf("the unrenameable path: %+v", left)
766 }
767 for i := 0; i < 2; i++ {
768 before = fences
769 a.tick()
770 if fences != before {
771 t.Fatal("a later pass fenced again for a path it cannot rename")
772 }
773 if again := wd.report(moves.OfflineRoleSource, "ie-a"); again.Phase != moves.OfflinePhaseParked || !reflect.DeepEqual(again.Retired, r.Retired) {
774 t.Fatalf("pass %d: %+v", i, again)
775 }
776 }
777 a.restart()
778 count()
779 before = fences
780 a.tick()
781 if fences != before || a.svc.Fenced(offHost) {
782 t.Fatal("after a restart the pass fenced again for a path it cannot rename")
783 }
784 if again := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(again.Retired, r.Retired) {
785 t.Fatalf("after a restart: %+v", again.Retired)
786 }
787 if _, err := os.Stat(filepath.Join(programs, offHost+".json")); err != nil {
788 t.Fatalf("the left path: %v", err)
789 }
790}
791
792// offline-open on a nested source: the contents of the raw instance
793// directory are renamed into the park directory, the directory and its
794// view stay, empty, and the inventory names each path.
795func TestOfflineOpenParksTheNestedSource(t *testing.T) {
796 wd := newOffWorld(t)
797 a := wd.box("ie-a", "", nil)
798 a.nested = true
799 raw := layout.RawInstanceDir(a.dd, offHost)
800 if err := os.MkdirAll(raw, 0o700); err != nil {
801 t.Fatal(err)
802 }
803 // The bind mount at <dd>/instances, as a link in the fixture.
804 if err := os.Symlink(filepath.Base(filepath.Dir(raw)), filepath.Join(a.dd, "instances")); err != nil {
805 t.Fatal(err)
806 }
807 a.tick()
808 rt := a.row(offHost)
809 if rt == nil || !provision.NestedPaths(a.dd, *rt) {
810 t.Fatalf("fixture: not a nested row: %+v", rt)
811 }
812 contents := map[string]string{
813 offHost + ".db": "sqlite",
814 offHost + ".db-shm": "shm",
815 offHost + ".env": "CARLOS_INSTANCE_KEY=k\n",
816 "instance.key": "key",
817 "uploads/a.png": "png",
818 }
819 for rel, body := range contents {
820 writeFile(t, filepath.Join(raw, rel), body)
821 }
822 wd.start()
823 a.tick()
824 if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.LocalDB || !r.Files || !r.Env {
825 t.Fatalf("nested report: %+v", r)
826 }
827 wd.seal()
828 wd.sealed()
829 wd.flip()
830 wd.landed()
831 wd.open()
832 a.tick()
833 park := moves.OfflineParkDir(a.dd, offHost, offID, moves.OfflineRoleSource)
834 for rel, body := range contents {
835 if got, err := os.ReadFile(filepath.Join(park, nestedParkPrefix, rel)); err != nil || string(got) != body {
836 t.Fatalf("park/instance/%s = %q %v", rel, got, err)
837 }
838 }
839 ents, err := os.ReadDir(raw)
840 if err != nil || len(ents) != 0 {
841 t.Fatalf("the instance directory: %v %v, want it there and empty", ents, err)
842 }
843 r := wd.report(moves.OfflineRoleSource, "ie-a")
844 if r.Phase != moves.OfflinePhaseParked || r.LocalDB || r.Files || r.Env {
845 t.Fatalf("parked report: %+v", r)
846 }
847 want := map[string]string{
848 filepath.Join(raw, offHost+".db"): moves.RetiredParked,
849 filepath.Join(raw, offHost+".db-wal"): moves.RetiredAbsent,
850 filepath.Join(raw, offHost+".db-shm"): moves.RetiredParked,
851 filepath.Join(raw, offHost+".db-litestream"): moves.RetiredAbsent,
852 filepath.Join(raw, ".activator-"+offHost+".db-litestream"): moves.RetiredAbsent,
853 filepath.Join(raw, offHost+".env"): moves.RetiredParked,
854 filepath.Join(raw, offHost+".config.env"): moves.RetiredAbsent,
855 filepath.Join(raw, "instance.key"): moves.RetiredParked,
856 filepath.Join(raw, "uploads"): moves.RetiredParked,
857 filepath.Join(a.dd, "programs", offHost+".json"): moves.RetiredAbsent,
858 provision.InstanceDir(a.dd, offHost): moves.RetiredLeft,
859 raw: moves.RetiredLeft,
860 filepath.Join(a.opt, "live", offHost): moves.RetiredAbsent,
861 }
862 assertRetired(t, r.Retired, want)
863 // Re-run: the same inventory, from what is parked.
864 a.tick()
865 if again := wd.report(moves.OfflineRoleSource, "ie-a"); !reflect.DeepEqual(again.Retired, r.Retired) {
866 t.Fatalf("second pass:\n %+v\n %+v", again.Retired, r.Retired)
867 }
868}
869
870// --- the dispatcher and the capability --------------------------------------
871
872// An offline record reaches offlineRound with Phase 1 off, ahead of the
873// Phase 1 skip and the lost-box gate: no host claim is read. A
874// Phase-1-shaped record with Phase 1 off is still logged and skipped.
875func TestOfflineDispatchPrecedesThePhase1Skip(t *testing.T) {
876 wd := newOffWorld(t)
877 wd.create("other.example.com")
878 a := wd.box("ie-a", "", nil)
879 a.tick()
880 wd.start()
881 // A Phase 1 move on the other host (its id is what makes it Phase 1).
882 if _, err := provision.SetRecordMove(context.Background(), wd.s, wd.acct, "app", "other.example.com", func(rec *provision.Record) error {
883 rec.Move = &provision.MoveState{ID: offID2, From: "", To: "uk", State: provision.MovePreparing, RequestedAt: wd.clock()}
884 return nil
885 }); err != nil {
886 t.Fatal(err)
887 }
888 a.tick()
889 if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); !ok {
890 t.Fatal("the offline record never reached offlineRound")
891 }
892 if a.w.lost != nil {
893 t.Fatalf("the lost-box gate ran: %v", a.w.lost)
894 }
895 if _, ok := a.w.notices.Load("other.example.com#switch"); !ok {
896 t.Fatal("the Phase 1 record was not logged as waiting on the switch")
897 }
898 if rt := a.row("other.example.com"); rt == nil || rt.HoldOp != "" || rt.Status != "ready" {
899 t.Fatalf("the Phase 1 record's row was touched: %+v", rt)
900 }
901 keys, err := wd.s.List(context.Background(), "control/offline-moves/other.example.com/")
902 if err != nil || len(keys) != 0 {
903 t.Fatalf("the Phase 1 record was reported as offline: %v %v", keys, err)
904 }
905}
906
907// The capability is the box's own witness: published with hibernation
908// configured and both switches unset, withheld otherwise. A box without it
909// runs no offline step and installs no offline hook: its row is never
910// held, nothing is reported, and one notice is logged. The guard mode is
911// not an input (platform-1e's ruling of 2026-10-20).
912func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) {
913 env := func(kv ...string) func(string) string {
914 m := map[string]string{"CARLOS_HIBERNATE_BUCKET": "data"}
915 for i := 0; i+1 < len(kv); i += 2 {
916 m[kv[i]] = kv[i+1]
917 }
918 return func(k string) string { return m[k] }
919 }
920 // The published capability is the armed worker's own answer (M5).
921 prev := armedMovesWorker.Load()
922 t.Cleanup(func() { armedMovesWorker.Store(prev) })
923 armedMovesWorker.Store(nil)
924 if armedOfflineCapable() {
925 t.Fatal("capable with no worker armed")
926 }
927 if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); len(got) != 0 {
928 t.Fatalf("no worker armed: %v", got)
929 }
930 wdCap := newOffWorld(t)
931 armed := wdCap.box("armed", "", nil)
932 armedMovesWorker.Store(armed.w)
933 if !armedOfflineCapable() {
934 t.Fatal("a capable armed worker does not publish")
935 }
936 if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); !slices.Equal(got, []string{capOfflineMove1, capOfflineStage1}) {
937 t.Fatalf("capable box: %v", got)
938 }
939 base := edgesFileCapabilities(env("CARLOS_REGION_HA", "1"))
940 if got := withOfflineCapability(base, true); !slices.Equal(got, []string{capRegionHA1, capOfflineMove1, capOfflineStage1}) || !slices.Equal(base, []string{capRegionHA1}) {
941 t.Fatalf("region HA too: %v (base %v)", got, base)
942 }
943 for _, kv := range [][]string{
944 {"CARLOS_HIBERNATE_BUCKET", ""},
945 {"CARLOS_HOST_CLAIM", "1"},
946 {"CARLOS_HOST_CLAIM", "0"}, // set at all
947 {moves.SwitchEnv, "on"},
948 {moves.SwitchEnv, "off"}, // set at all
949 } {
950 if offlineCapable(env(kv...), hibernationConfigured(env(kv...))) {
951 t.Fatalf("%s=%q is capable", kv[0], kv[1])
952 }
953 incapable := wdCap.box("incapable", "", map[string]string{kv[0]: kv[1]})
954 armedMovesWorker.Store(incapable.w)
955 if got := withOfflineCapability(edgesFileCapabilities(env(kv...)), armedOfflineCapable()); slices.Contains(got, capOfflineMove1) || slices.Contains(got, capOfflineStage1) {
956 t.Fatalf("%s=%q publishes: %v", kv[0], kv[1], got)
957 }
958 }
959 armedMovesWorker.Store(prev)
960
961 wd := newOffWorld(t)
962 a := wd.box("ie-a", "", map[string]string{"CARLOS_HOST_CLAIM": "1"})
963 if a.w.cfg.OfflineCapable {
964 t.Fatal("armOffline: a claims box is capable")
965 }
966 a.tick()
967 wd.start()
968 a.tick()
969 if rt := a.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
970 t.Fatalf("an incapable box held the row: %+v", rt)
971 }
972 if a.svc.Fenced(offHost) {
973 t.Fatal("an incapable box fenced")
974 }
975 if _, ok := wd.reportIf(moves.OfflineRoleSource, "ie-a"); ok {
976 t.Fatal("an incapable box reported")
977 }
978 if _, ok := a.w.notices.Load(offHost + "#offline"); !ok {
979 t.Fatal("no notice from an incapable box")
980 }
981}
982
983// --- dormant ---------------------------------------------------------------
984
985// flatLayout is a reconcile config's layout read, always flat.
986func flatLayout(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil }
987
988// dormantWorld is a deployment with no offline record anywhere: offHost at
989// rest, gone.example.com (whose record the test deletes), a legacy move and
990// a Phase 1 move on records of their own.
991func dormantWorld(t *testing.T) *offWorld {
992 t.Helper()
993 wd := newOffWorld(t)
994 for _, h := range []string{"gone.example.com", "legacy.example.com", "phase1.example.com"} {
995 wd.create(h)
996 }
997 wd.moveHost("legacy.example.com", func(rec *provision.Record) error {
998 rec.Move = &provision.MoveState{To: "uk", State: provision.MovePreparing}
999 return nil
1000 })
1001 wd.moveHost("phase1.example.com", func(rec *provision.Record) error {
1002 rec.Move = &provision.MoveState{ID: offID2, From: "", To: "uk", State: provision.MovePreparing, RequestedAt: wd.clock()}
1003 return nil
1004 })
1005 return wd
1006}
1007
1008// dormantBoxes are two home boxes minted from the same records with no
1009// hooks at all, so their registries start equal: the plain one stands for
1010// main, the hooked one has a capable worker whose view no listing has read.
1011func dormantBoxes(t *testing.T, wd *offWorld) (plain, hooked *offBox) {
1012 t.Helper()
1013 plain, hooked = wd.box("main", "", nil), wd.box("hooked", "", nil)
1014 for _, b := range []*offBox{plain, hooked} {
1015 if err := reconcileSerializedWith(context.Background(), wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, nil); err != nil {
1016 t.Fatal(err)
1017 }
1018 if b.row("gone.example.com") == nil {
1019 t.Fatalf("fixture: %s has no row to retire", b.name)
1020 }
1021 }
1022 if hooked.w.cfg.Offline.built() || !hooked.w.cfg.OfflineCapable {
1023 t.Fatal("fixture: the hooked box's view is built, or the box is not capable")
1024 }
1025 return plain, hooked
1026}
1027
1028// sameRegistries fails unless the two boxes' registries match, paths and
1029// creation times aside (each box has its own data directory and clock).
1030func sameRegistries(t *testing.T, a, b *offBox) {
1031 t.Helper()
1032 strip := func(rs []registry.Route) []registry.Route {
1033 out := append([]registry.Route(nil), rs...)
1034 for i := range out {
1035 out[i].DBPath, out[i].Socket, out[i].CreatedAt = "", "", 0
1036 }
1037 return out
1038 }
1039 ar, err := a.reg.List()
1040 if err != nil {
1041 t.Fatal(err)
1042 }
1043 br, err := b.reg.List()
1044 if err != nil {
1045 t.Fatal(err)
1046 }
1047 if !reflect.DeepEqual(strip(ar), strip(br)) {
1048 t.Fatalf("the registries differ:\n %s %+v\n %s %+v", a.name, ar, b.name, br)
1049 }
1050}
1051
1052// With no offline record anywhere (a legacy and a Phase 1 move included):
1053// the very first reconcile on a box whose view no listing has read removes
1054// a retired row exactly as main does; after it the hold answers false for
1055// every record and keep false for every host; and the edges file differs
1056// by the one capability string and nothing else.
1057func TestOfflineDormantIsMain(t *testing.T) {
1058 ctx := context.Background()
1059 wd := dormantWorld(t)
1060 plain, hooked := dormantBoxes(t, wd)
1061 if err := wd.s.Delete(ctx, provision.RecordKey(wd.acct, "app", "gone.example.com")); err != nil {
1062 t.Fatal(err)
1063 }
1064 for _, b := range []*offBox{plain, hooked} {
1065 w := b.w
1066 if b == plain {
1067 w = nil
1068 }
1069 if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
1070 t.Fatal(err)
1071 }
1072 if rt := b.row("gone.example.com"); rt != nil {
1073 t.Fatalf("%s kept a retired row on its first pass: %+v", b.name, rt)
1074 }
1075 }
1076 sameRegistries(t, plain, hooked)
1077 v := hooked.w.cfg.Offline
1078 if !v.built() || v.failed() {
1079 t.Fatal("the first reconcile did not read the view")
1080 }
1081 for _, h := range []string{offHost, "gone.example.com", "legacy.example.com", "phase1.example.com", "never.example.com"} {
1082 if v.keep(h) {
1083 t.Fatalf("keep(%s)", h)
1084 }
1085 }
1086 recs, err := provision.ListAllRecords(ctx, wd.s)
1087 if err != nil {
1088 t.Fatal(err)
1089 }
1090 if len(recs) != 3 {
1091 t.Fatalf("fixture: %d records", len(recs))
1092 }
1093 for _, rec := range recs {
1094 if _, _, ok := offlineHold(hooked.w.cfg)(rec); ok {
1095 t.Fatalf("held %s", rec.Host)
1096 }
1097 }
1098 // A further pass changes nothing on either box.
1099 for _, b := range []*offBox{plain, hooked} {
1100 w := b.w
1101 if b == plain {
1102 w = nil
1103 }
1104 if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
1105 t.Fatal(err)
1106 }
1107 }
1108 sameRegistries(t, plain, hooked)
1109
1110 // The edges file: the one string, and no headroom with it.
1111 var last1, last2 []byte
1112 none := func(string) string { return "" }
1113 plainFile := edgesFile{Box: "plain", Placement: "", Capabilities: withOfflineCapability(edgesFileCapabilities(none), false)}
1114 capable := edgesFile{Box: "plain", Placement: "", Capabilities: withOfflineCapability(edgesFileCapabilities(none), true)}
1115 if headroomCapabilities(capable.Capabilities) != nil {
1116 t.Fatal("offline-move-1 asks for headroom")
1117 }
1118 s1, s2 := release.NewDirStore(t.TempDir()), release.NewDirStore(t.TempDir())
1119 if err := edgesFileWrite(ctx, s1, plain.reg, plainFile, "1.2.3.4", &last1, false); err != nil {
1120 t.Fatal(err)
1121 }
1122 if err := edgesFileWrite(ctx, s2, plain.reg, capable, "1.2.3.4", &last2, false); err != nil {
1123 t.Fatal(err)
1124 }
1125 b1, _, _ := s1.Get(ctx, edgesFileKey("plain"))
1126 b2, _, _ := s2.Get(ctx, edgesFileKey("plain"))
1127 var m1, m2 map[string]any
1128 if json.Unmarshal(b1, &m1) != nil || json.Unmarshal(b2, &m2) != nil {
1129 t.Fatal("edges files do not parse")
1130 }
1131 if caps, _ := m2["capabilities"].([]any); len(caps) != 2 || caps[0] != capOfflineMove1 || caps[1] != capOfflineStage1 {
1132 t.Fatalf("capabilities: %v", m2["capabilities"])
1133 }
1134 delete(m2, "capabilities")
1135 delete(m1, "generation")
1136 delete(m2, "generation")
1137 if !reflect.DeepEqual(m1, m2) {
1138 t.Fatalf("the edges file differs beyond the capability:\n %s\n %s", b1, b2)
1139 }
1140 if bytes.Contains(b1, []byte("capabilities")) {
1141 t.Fatal("fixture: the plain file has capabilities")
1142 }
1143}
1144
1145// M1: a failed startup listing keeps every row only until the next
1146// reconcile, which reads the listing again: the retired row then goes as on
1147// main, without waiting for a moves pass.
1148func TestOfflineFailedListingHealsOnTheNextReconcile(t *testing.T) {
1149 ctx := context.Background()
1150 wd := dormantWorld(t)
1151 plain, hooked := dormantBoxes(t, wd)
1152 v := hooked.w.cfg.Offline
1153 refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, v)
1154 if !v.built() || !v.failed() || !v.keep(offHost) {
1155 t.Fatal("a failed startup listing does not keep")
1156 }
1157 if err := wd.s.Delete(ctx, provision.RecordKey(wd.acct, "app", "gone.example.com")); err != nil {
1158 t.Fatal(err)
1159 }
1160 for _, b := range []*offBox{plain, hooked} {
1161 w := b.w
1162 if b == plain {
1163 w = nil
1164 }
1165 if err := reconcileSerializedWith(ctx, wd.s, b.reg, provision.ReconcileConfig{DataDir: b.dd, Layout: flatLayout}, w); err != nil {
1166 t.Fatal(err)
1167 }
1168 }
1169 if v.failed() || v.keep("gone.example.com") {
1170 t.Fatal("the reconcile did not read the listing again")
1171 }
1172 if rt := hooked.row("gone.example.com"); rt != nil {
1173 t.Fatalf("the retired row stayed after the view healed: %+v", rt)
1174 }
1175 sameRegistries(t, plain, hooked)
1176 // While the listing keeps failing the view stays closed.
1177 refreshOfflineView(ctx, failingListStore{Store: wd.s, prefix: "apps/", err: errors.New("listing refused")}, v)
1178 if !v.keep(offHost) {
1179 t.Fatal("a failed listing does not keep")
1180 }
1181 unbuilt := newOfflineView(moveCfg{Region: ""}, nil)
1182 if !unbuilt.keep(offHost) {
1183 t.Fatal("an unbuilt view keeps nothing")
1184 }
1185}
1186
1187// M4: a record that does not decode is kept by keep alone (its move cannot
1188// be known), logged, and every other host is judged as usual. The moves
1189// pass's listing carries the same answer.
1190func TestOfflineUndecodableRecordIsKept(t *testing.T) {
1191 ctx := context.Background()
1192 wd := newOffWorld(t)
1193 wd.create("broken.example.com")
1194 if _, err := wd.s.Put(ctx, provision.RecordKey(wd.acct, "app", "broken.example.com"), []byte("{not json"), mustETag(t, wd.s, provision.RecordKey(wd.acct, "app", "broken.example.com"))); err != nil {
1195 t.Fatal(err)
1196 }
1197 a := wd.box("ie-a", "", nil)
1198 refreshOfflineView(ctx, wd.s, a.w.cfg.Offline)
1199 v := a.w.cfg.Offline
1200 if v.failed() || !v.keep("broken.example.com") || v.keep(offHost) {
1201 t.Fatalf("startup view: failed %v, keep(broken) %v, keep(%s) %v", v.failed(), v.keep("broken.example.com"), offHost, v.keep(offHost))
1202 }
1203 fresh := newOfflineView(a.w.cfg, nil)
1204 a.w.cfg.Offline = fresh
1205 a.pass()
1206 if !fresh.keep("broken.example.com") || fresh.keep(offHost) {
1207 t.Fatal("the moves pass's listing does not keep the undecodable host")
1208 }
1209 if _, _, ok := offlineHold(a.w.cfg)(provision.Record{Host: "broken.example.com"}); ok {
1210 t.Fatal("held a record with no move")
1211 }
1212}
1213
1214func mustETag(t *testing.T, s release.Store, key string) string {
1215 t.Helper()
1216 _, etag, err := s.Get(context.Background(), key)
1217 if err != nil {
1218 t.Fatal(err)
1219 }
1220 return etag
1221}
1222
1223// hostSliceCgroup is the slice instancectl's units run in.
1224func TestOfflineHostSliceIsInstancectlsSlice(t *testing.T) {
1225 got := hostSliceCgroup("/sys/fs/cgroup", offHost)
1226 want := filepath.Join("/sys/fs/cgroup", "carlos.slice", "carlos-instances.slice", instancectl.SliceName(offHost))
1227 if got != want {
1228 t.Fatalf("hostSliceCgroup = %s, want %s", got, want)
1229 }
1230 if filepath.Dir(filepath.Join("/sys/fs/cgroup", filepath.FromSlash(instancectl.ClaimUnitControlGroup(offHost)))) != got {
1231 t.Fatal("the run unit is not in the slice")
1232 }
1233}
1234
1235// The seal record's writer is create-only: the first body stays, a re-run
1236// gets it back and writes nothing, and the report writer beside it replaces.
1237func TestOfflineCreateOnlyKeepsTheFirstWrite(t *testing.T) {
1238 ctx := context.Background()
1239 s := release.NewDirStore(t.TempDir())
1240 key := moves.OfflineReportPrefix(offHost, offID) + "seal.json"
1241 got, created, err := putOfflineCreateOnly(ctx, s, key, []byte(`{"n":1}`))
1242 if err != nil || !created || string(got) != `{"n":1}` {
1243 t.Fatalf("first write: %q %v %v", got, created, err)
1244 }
1245 got, created, err = putOfflineCreateOnly(ctx, s, key, []byte(`{"n":2}`))
1246 if err != nil || created || string(got) != `{"n":1}` {
1247 t.Fatalf("second write: %q %v %v, want the first body back, nothing written", got, created, err)
1248 }
1249 if body, _, err := s.Get(ctx, key); err != nil || string(body) != `{"n":1}` {
1250 t.Fatalf("stored: %q %v", body, err)
1251 }
1252 if _, _, err := putOfflineCreateOnly(ctx, s, "", nil); err == nil {
1253 t.Fatal("an empty key was written")
1254 }
1255}
1256
1257// keep's table: the source keeps until finishing, the target until the
1258// move is cleared, a third region never.
1259func TestOfflineKeep(t *testing.T) {
1260 rec := func(region, state string) provision.Record {
1261 return provision.Record{Host: offHost, Region: region, Move: &provision.MoveState{Kind: provision.MoveKindOffline, ID: offID, From: "", To: "uk", State: state}}
1262 }
1263 states := []string{provision.OfflineStopping, provision.OfflineSealing, provision.OfflineSealed, provision.OfflineLanding,
1264 provision.OfflineLanded, provision.OfflineOpen, provision.OfflineFinishing, provision.OfflineReverting}
1265 for _, st := range states {
1266 recs := []provision.Record{rec("uk", st)}
1267 src := newOfflineView(moveCfg{Region: ""}, nil)
1268 src.refresh(recs, nil)
1269 if got, want := src.keep(offHost), st != provision.OfflineFinishing; got != want {
1270 t.Fatalf("source at %s: keep %v", st, got)
1271 }
1272 tgt := newOfflineView(moveCfg{Region: "uk"}, nil)
1273 tgt.refresh(recs, nil)
1274 if !tgt.keep(offHost) {
1275 t.Fatalf("target at %s: not kept", st)
1276 }
1277 other := newOfflineView(moveCfg{Region: "fr"}, nil)
1278 other.refresh(recs, nil)
1279 if other.keep(offHost) {
1280 t.Fatalf("a third region keeps at %s", st)
1281 }
1282 }
1283}
1284
1285// readCountingStore counts the reads made through it.
1286type readCountingStore struct {
1287 release.Store
1288 mu sync.Mutex
1289 reads int
1290}
1291
1292func (c *readCountingStore) Get(ctx context.Context, key string) ([]byte, string, error) {
1293 c.mu.Lock()
1294 c.reads++
1295 c.mu.Unlock()
1296 return c.Store.Get(ctx, key)
1297}
1298
1299func (c *readCountingStore) List(ctx context.Context, prefix string) ([]string, error) {
1300 c.mu.Lock()
1301 c.reads++
1302 c.mu.Unlock()
1303 return c.Store.List(ctx, prefix)
1304}
1305
1306// Harness Run 9, finding 7: between moves passes, a source box rewrites its
1307// report on the re-assert tick while a door judges that report by age
1308// (stopping to landed), and only then. It never runs the seal itself, it
1309// reads nothing while no offline move is in flight, and a box that is
1310// neither a source nor the sealer does nothing.
1311func TestOfflineReassertRewritesOnlyTheReportsADoorReads(t *testing.T) {
1312 wd := newOffWorld(t)
1313 a := wd.box("ie-a", "", nil)
1314 uk := wd.box("uk-box", "uk", nil)
1315 a.tick()
1316 uk.tick()
1317 cs := &readCountingStore{Store: wd.s}
1318 ctx := context.Background()
1319 if n := a.w.offlineReassert(ctx, cs); n != 0 || cs.reads != 0 {
1320 t.Fatalf("no move in flight: acted on %d, %d store reads", n, cs.reads)
1321 }
1322
1323 wd.start()
1324 a.tick() // the start door's bump
1325 uk.tick()
1326 wd.advance(3 * time.Minute)
1327 for _, state := range []string{provision.OfflineStopping, provision.OfflineSealing} {
1328 if state == provision.OfflineSealing {
1329 wd.seal()
1330 wd.advance(time.Minute)
1331 }
1332 if n := a.w.offlineReassert(ctx, cs); n != 1 {
1333 t.Fatalf("%s: the source acted on %d records, want 1", state, n)
1334 }
1335 if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.At.Equal(wd.clock()) || !r.Held || !r.Fenced {
1336 t.Fatalf("%s: the source's report %+v, want rewritten at %v", state, r, wd.clock())
1337 }
1338 if n := uk.w.offlineReassert(ctx, cs); n != 0 {
1339 t.Fatalf("%s: the target acted on %d records", state, n)
1340 }
1341 }
1342 if _, _, err := wd.s.Get(ctx, moves.OfflineSealKey(offHost, offID)); !errors.Is(err, os.ErrNotExist) {
1343 t.Fatalf("the re-assert ran the seal (err %v)", err)
1344 }
1345
1346 // From open on, no door reads a report by age: nothing is rewritten.
1347 wd.sealed()
1348 wd.flip()
1349 wd.landed()
1350 wd.open()
1351 a.pass() // the view learns the state the way it always does
1352 at := wd.report(moves.OfflineRoleSource, "ie-a").At
1353 wd.advance(time.Minute)
1354 if n := a.w.offlineReassert(ctx, cs); n != 0 {
1355 t.Fatalf("open: the source acted on %d records", n)
1356 }
1357 if r := wd.report(moves.OfflineRoleSource, "ie-a"); !r.At.Equal(at) {
1358 t.Fatalf("open: the source's report was rewritten at %v", r.At)
1359 }
1360}
1361
1362// elsewhere, scheduleSkip's second half (harness Run 9, defect 8): the box
1363// whose region the record does not name, through every state the record
1364// moves through.
1365func TestOfflineElsewhereFollowsTheRecordsRegion(t *testing.T) {
1366 wd := newOffWorld(t)
1367 a := wd.box("ie-a", "", nil)
1368 uk := wd.box("uk-box", "uk", nil)
1369 a.tick()
1370 uk.tick()
1371 check := func(when string, ieElsewhere, ukElsewhere bool) {
1372 t.Helper()
1373 a.pass()
1374 uk.pass()
1375 if got := a.w.cfg.Offline.elsewhere(offHost); got != ieElsewhere {
1376 t.Fatalf("%s: ie-a elsewhere = %v", when, got)
1377 }
1378 if got := uk.w.cfg.Offline.elsewhere(offHost); got != ukElsewhere {
1379 t.Fatalf("%s: uk-box elsewhere = %v", when, got)
1380 }
1381 }
1382 check("no move", false, false)
1383 wd.start()
1384 check("stopping", false, true)
1385 wd.seal()
1386 wd.sealed()
1387 check("sealed", false, true)
1388 wd.flip()
1389 check("landing", true, false)
1390 wd.abort()
1391 check("reverting", false, true)
1392}
1393