amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1//go:build linux
2
3package host
4
5import (
6 "bytes"
7 "context"
8 "crypto/ed25519"
9 "encoding/json"
10 "errors"
11 "io/fs"
12 "net/http"
13 "net/http/httptest"
14 "os"
15 "path/filepath"
16 "reflect"
17 "sort"
18 "strings"
19 "testing"
20 "time"
21
22 "crypto/sha256"
23 "encoding/hex"
24
25 "github.com/carlosframework/platform/internal/activator"
26 "github.com/carlosframework/platform/internal/appconfig"
27 "github.com/carlosframework/platform/internal/cgroup"
28 "github.com/carlosframework/platform/internal/hostwrite"
29 "github.com/carlosframework/platform/internal/instancekey"
30 "github.com/carlosframework/platform/internal/lease"
31 "github.com/carlosframework/platform/internal/lease/leasetest"
32 "github.com/carlosframework/platform/internal/moves"
33 "github.com/carlosframework/platform/internal/provision"
34 "github.com/carlosframework/platform/internal/registry"
35 "github.com/carlosframework/platform/internal/release"
36)
37
38// The offline move's landing, open on the target and the target's revert
39// (Task 5 of docs/superpowers/plans/2026-10-20-offline-operator-move.md),
40// tested through the functions the agent calls (newMovesWorker, armOffline,
41// reconcileSerializedWith, the edges-file writer and movesPass), over one
42// DirStore for the deployment bucket, leasetest's S3 server for the data
43// buckets data-ie and data-uk, real registries, a real litestream replica,
44// the real archive, the real held render and the exec backend's real launch
45// composition.
46//
47// The seal is Task 4's: the source box ie-a runs the sealer step
48// (offlineSealerStep, through offlineRound) over T4's seal world, so the
49// staged replica, the archive, its manifest, the timers snapshot, the env
50// report and seal.json are exactly what the sealer writes.
51
52const (
53 landKeyValue = "the-instance-key" // the key sealWorld.instance writes
54 landStore = "shop-store-ie"
55 landSecret = "store-secret-value"
56)
57
58// landS3 reads the data buckets' model directly.
59type landS3 struct{ f *leasetest.VersionedBucket }
60
61// current is the current objects under prefix in bucket.
62func (s *landS3) current(t *testing.T, bucket, prefix string) map[string][]byte {
63 t.Helper()
64 vers, err := s.f.ListVersions(context.Background(), bucket, prefix)
65 if err != nil {
66 t.Fatal(err)
67 }
68 out := map[string][]byte{}
69 for _, v := range vers {
70 if v.IsLatest && !v.DeleteMarker {
71 body, _ := s.f.Current(bucket, v.Key)
72 out[v.Key] = body
73 }
74 }
75 return out
76}
77
78// landOpts shapes the source: configKey sets the instance key in the app's
79// configuration (and none in <host>.env), noFiles gives the host no files
80// directory, nested makes it a nested row with the launch descriptors a real
81// nested host keeps.
82type landOpts struct {
83 configKey string
84 noFiles bool
85 nested bool
86}
87
88// landWorld is offHost at home (ie-a, data-ie) moving to uk (uk-box,
89// data-uk), with a real database replica, a files directory, an env file
90// carrying the instance key, an app config bundle carrying the store
91// binding, timers, a NextDue, a replication rung and a promoted release,
92// sealed by the real sealer.
93type landWorld struct {
94 t *testing.T
95 sw *sealWorld
96 wd *offWorld
97 s3 *landS3
98 ie *offBox
99 uk *offBox
100 ukEB *activator.ExecBackend
101
102 relPriv ed25519.PrivateKey
103
104 seal moves.OfflineReport
105 sealRaw []byte
106 src provision.MoveSource
107
108 opts landOpts
109 // The source's files directory, env file and config file.
110 srcFiles, srcEnv, srcConfig string
111}
112
113func newLandWorld(t *testing.T) *landWorld { return newLandWorldWith(t, landOpts{}) }
114
115func newLandWorldWith(t *testing.T, opts landOpts) *landWorld {
116 t.Helper()
117 ctx := context.Background()
118 sw := newSealWorld(t)
119 lw := &landWorld{t: t, sw: sw, wd: sw.offWorld, s3: &landS3{f: sw.fake}, ie: sw.ie, uk: sw.uk, opts: opts}
120 seed, pub, err := release.GenerateKey()
121 if err != nil {
122 t.Fatal(err)
123 }
124 if lw.relPriv, err = release.ParseSigningKey(seed, "test"); err != nil {
125 t.Fatal(err)
126 }
127 shipVersion(t, sw.s, sw.acct, "app", sealVersion)
128 shipVersion(t, sw.s, sw.acct, "app", "v2")
129 lw.promote(sealVersion)
130 lw.bundle(landStore)
131 lw.uk.env["CARLOS_HIBERNATE_REGION"] = sealUKRegion
132 lw.uk.env["CARLOS_RELEASE_PUBKEY"] = pub
133 lw.ukEB = &activator.ExecBackend{DataDir: lw.uk.dd, OptDir: lw.uk.opt, Region: "uk"}
134 lw.arm(lw.uk)
135
136 // The source at rest, as a box that has served it.
137 ie := lw.ie
138 sw.instance(ie)
139 lw.srcFiles, lw.srcEnv, lw.srcConfig = filepath.Join(ie.dd, offHost), filepath.Join(ie.dd, offHost+".env"), filepath.Join(ie.dd, offHost+".config.env")
140 if opts.nested {
141 raw := sw.nest(ie)
142 lw.srcFiles, lw.srcEnv, lw.srcConfig = filepath.Join(raw, offHost), filepath.Join(raw, offHost+".env"), filepath.Join(raw, offHost+".config.env")
143 for _, name := range []string{"launch.run.json", "launch.seed.json", "launch.gen.7.json"} {
144 writeFile(t, filepath.Join(raw, name), "{}")
145 }
146 // The source's env pin is composed from the instance directory's
147 // own env and config files, as the agent's exec backend reads them
148 // through the instance mount. The fixture's mount is a symlink,
149 // which the nested reader refuses (and the seal world's backend,
150 // with no socket directory, classifies the row foreign and reads
151 // the flat paths), so here the raw directory stands in for the
152 // mount: the same two files, composed by the same code.
153 ie.w.cfg.LaunchLayers = func(rt registry.Route) (map[string]string, map[string]bool, []string, error) {
154 rt.DBPath, rt.Socket = provision.DefaultDBPath(raw, rt.Host), filepath.Join(provision.SocketDir, rt.Host+".sock")
155 return (&activator.ExecBackend{DataDir: raw}).LaunchLayers(rt)
156 }
157 }
158 if opts.configKey != "" {
159 writeFile(t, lw.srcEnv, "CARLOS_ADMIN_TOKEN=token\n")
160 }
161 if opts.noFiles {
162 if err := os.RemoveAll(lw.srcFiles); err != nil {
163 t.Fatal(err)
164 }
165 }
166
167 // start and the source's hold; its config rendered held from the bundle
168 // (as SyncExec renders it at rest); seal; the sealer's pass.
169 sw.startAndSeal("ie-a", ie)
170 hr, err := ie.w.cfg.RenderHeld(ctx, sw.s, *ie.row(offHost), offID, nil, func(string) error { return nil })
171 if err != nil {
172 t.Fatal(err)
173 }
174 if hr.Path != lw.srcConfig {
175 // The nested fixture's sockets are not production's, so the render
176 // took the flat path; production renders into the instance
177 // directory, which is where the row's launch reads it.
178 if err := os.Rename(hr.Path, lw.srcConfig); err != nil {
179 t.Fatal(err)
180 }
181 }
182 sw.mustPass(ie)
183 rec := sw.record()
184 if rec.Move.State != provision.OfflineSealed {
185 t.Fatalf("not sealed: %+v (%+v)", rec.Move, sw.report(moves.OfflineRoleSealer, "ie-a").Items)
186 }
187 var ok bool
188 if lw.seal, lw.sealRaw, ok = sw.sealRecord(); !ok {
189 t.Fatal("no seal.json")
190 }
191 lw.src = *rec.Move.Source
192 return lw
193}
194
195// arm wires the target box as armMovesPass would, over its own data bucket.
196func (lw *landWorld) arm(b *offBox) {
197 bucket, region := b.getenv("CARLOS_HIBERNATE_BUCKET"), b.getenv("CARLOS_HIBERNATE_REGION")
198 b.svc = activator.New(activator.Config{Bucket: bucket, Idle: time.Hour}, &fakeWakeBackend{}, b.reg)
199 cfg := moveCfg{Region: b.region, Box: b.name, DataDir: b.dd, OptDir: b.opt, Quiesced: b.svc.Quiesced, Route: b.reg.Get, Now: b.wd.clock,
200 OwnBucket: bucket, OwnBucketRegion: region, OwnPrefix: sealPrefix,
201 Mirror: func(ctx context.Context, dst string) (lease.MirrorStore, error) {
202 return lease.NewS3Mirror(ctx, region, dst)
203 }}
204 armOffline(&cfg, b.dd, b.reg, b.svc, nil, hostwrite.Config{}, b.getenv)
205 cfg.Offline = newOfflineView(cfg, nil)
206 cfg.ReleaseHold = releaseHeld(b.reg, nil)
207 cfg.Procs = func(host string) (int, error) { return cgroup.HostProcs(b.cg, host, "") }
208 cfg.LaunchLayers = lw.ukEB.LaunchLayers
209 b.w = newMovesWorker(cfg)
210}
211
212// restart is an agent restart on the target: a fresh activator and worker
213// over the same registry, data directory and buckets.
214func (lw *landWorld) restart() { lw.arm(lw.uk) }
215
216func (lw *landWorld) promote(v string) {
217 lw.t.Helper()
218 if err := release.Promote(context.Background(), lw.wd.s, lw.relPriv, lw.wd.acct, "app", "stable", v); err != nil {
219 lw.t.Fatal(err)
220 }
221}
222
223// bundle writes the app's config: the store binding, its secret and a
224// setting, and the instance key when the world's configuration sets it.
225func (lw *landWorld) bundle(store string) {
226 lw.t.Helper()
227 env, err := appconfig.SealBound(lw.sw.priv.PublicKey(), appconfig.Binding{Account: lw.wd.acct, App: "app"}, []byte(landSecret))
228 if err != nil {
229 lw.t.Fatal(err)
230 }
231 m := appconfig.Manifest{Schema: 1, App: "app", Vars: map[string]appconfig.Var{
232 "STORE_BUCKET": {Kind: "env", Value: store},
233 "STORE_ENDPOINT": {Kind: "env", Value: "https://store.example.com"},
234 "STORE_SECRET_KEY": {Kind: "secret", Envelope: env},
235 "FEATURE": {Kind: "env", Value: "on"},
236 }}
237 if lw.opts.configKey != "" {
238 m.Vars[instancekey.Var] = appconfig.Var{Kind: "env", Value: lw.opts.configKey}
239 }
240 if err := moves.PutReplace(context.Background(), lw.wd.s, appconfig.Key(lw.wd.acct, "app"), mustJSON(lw.t, m)); err != nil {
241 lw.t.Fatal(err)
242 }
243}
244
245// flipAndLand flips the record and runs the target box's ticks.
246func (lw *landWorld) flipAndLand(ticks int) {
247 lw.wd.flip()
248 for range ticks {
249 lw.uk.tick()
250 }
251}
252
253func (lw *landWorld) report() moves.OfflineReport {
254 lw.t.Helper()
255 return lw.wd.report(moves.OfflineRoleTarget, "uk-box")
256}
257
258func (lw *landWorld) state() string {
259 lw.t.Helper()
260 rec := lw.wd.record()
261 if rec.Move == nil {
262 return ""
263 }
264 return rec.Move.State
265}
266
267// requireLanded is the happy landing's state: offline-landed, every landing
268// item OK in a landed report, the row held landing by the move, the edges
269// file not listing the host, and nothing ever taking the target's lease.
270func (lw *landWorld) requireLanded() {
271 t := lw.t
272 t.Helper()
273 if st := lw.state(); st != provision.OfflineLanded {
274 t.Fatalf("state %q, want %q; report %+v", st, provision.OfflineLanded, lw.report().Items)
275 }
276 r := lw.report()
277 if ok, bad := r.ItemsOK(moves.OfflineOwnerLanding); !ok || r.Phase != moves.OfflinePhaseLanded {
278 t.Fatalf("phase %s, not OK: %v (%+v)", r.Phase, bad, r.Items)
279 }
280 if len(r.Items) != len(moves.OfflineOwned(moves.OfflineOwnerLanding)) {
281 t.Fatalf("report carries %d items, want every landing item", len(r.Items))
282 }
283 if !reflect.DeepEqual(r.Staged, lw.seal.Staged) {
284 t.Fatalf("the report's verified listing %v, the seal's %v", keysOf(r.Staged), keysOf(lw.seal.Staged))
285 }
286 rt := lw.uk.row(offHost)
287 if rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
288 t.Fatalf("target row %+v", rt)
289 }
290 lw.uk.edges()
291 if lw.uk.lists(offHost) {
292 t.Fatal("the edges file lists the host before open")
293 }
294 lw.requireNoTargetLease()
295}
296
297// requireNoTargetLease: nothing has written under data-uk instances/ but,
298// at most, a lifted tombstone (a stale one the landing lifted), and the
299// host's lock is free.
300func (lw *landWorld) requireNoTargetLease() {
301 lw.t.Helper()
302 loc := activator.HibernatePrefix(sealPrefix, offHost)
303 for k := range lw.s3.current(lw.t, sealUK, "instances/") {
304 if k != loc+"/lock.json" {
305 lw.t.Fatalf("data-uk holds %s before the first wake", k)
306 }
307 }
308 _, owner, expires, err := lease.LockState(context.Background(), sealUK, loc, sealUKRegion)
309 if err != nil || !(owner == "" || lease.IsLiftedTombstone(owner, expires)) {
310 lw.t.Fatalf("the target's lock is %q until %v (%v)", owner, expires, err)
311 }
312}
313
314func keysOf[V any](m map[string]V) []string {
315 out := make([]string, 0, len(m))
316 for k := range m {
317 out = append(out, k)
318 }
319 sort.Strings(out)
320 return out
321}
322
323// tree is every regular file and symlink under dir with its content's
324// sha256, every directory with its mode: what a pass may not change. The
325// registry is left out (its file changes on every reconcile).
326func tree(t *testing.T, dir string) map[string]string {
327 t.Helper()
328 out := map[string]string{}
329 err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
330 if err != nil {
331 return err
332 }
333 rel, _ := filepath.Rel(dir, p)
334 if strings.HasPrefix(rel, "registry.db") {
335 return nil
336 }
337 info, err := d.Info()
338 if err != nil {
339 return err
340 }
341 switch {
342 case d.IsDir():
343 out[rel+"/"] = info.Mode().String()
344 case info.Mode().IsRegular():
345 b, err := os.ReadFile(p)
346 if err != nil {
347 return err
348 }
349 sum := sha256.Sum256(b)
350 out[rel] = hex.EncodeToString(sum[:]) + " " + info.Mode().String()
351 default:
352 out[rel] = info.Mode().String()
353 }
354 return nil
355 })
356 if err != nil {
357 t.Fatal(err)
358 }
359 return out
360}
361
362// requireSourceBytes: the target's files directory, env file and config
363// file are the source's byte for byte. A source with no files directory
364// lands an empty one.
365func (lw *landWorld) requireSourceBytes() {
366 t := lw.t
367 t.Helper()
368 dst := tree(t, filepath.Join(lw.uk.dd, offHost))
369 if lw.opts.noFiles {
370 if len(dst) != 1 || dst["./"] == "" {
371 t.Fatalf("a source with no files directory landed %v", dst)
372 }
373 } else if src := tree(t, lw.srcFiles); !reflect.DeepEqual(src, dst) {
374 t.Fatalf("files differ:\nsource %v\ntarget %v", src, dst)
375 }
376 for src, name := range map[string]string{lw.srcEnv: offHost + ".env", lw.srcConfig: offHost + ".config.env"} {
377 a, err := os.ReadFile(src)
378 if err != nil {
379 t.Fatal(err)
380 }
381 b, err := os.ReadFile(filepath.Join(lw.uk.dd, name))
382 if err != nil {
383 t.Fatalf("%s on the target: %v", name, err)
384 }
385 if !bytes.Equal(a, b) {
386 t.Fatalf("%s differs:\nsource %q\ntarget %q", name, a, b)
387 }
388 }
389}
390
391func itemsSummary(r moves.OfflineReport) map[string]moves.Check { return r.Items }
392
393func (lw *landWorld) parked(role string) map[string]string {
394 dir := moves.OfflineParkDir(lw.uk.dd, offHost, offID, role)
395 if !isDir(dir) {
396 return nil
397 }
398 return tree(lw.t, dir)
399}
400
401// crashAt makes the target's landing crash once at point.
402func (lw *landWorld) crashAt(point string) *int {
403 crashed := 0
404 lw.uk.w.cfg.landFault = func(p string) error {
405 if p == point && crashed == 0 {
406 crashed++
407 return errors.New("crash")
408 }
409 return nil
410 }
411 return &crashed
412}
413
414// --- (a) and (f): the landing --------------------------------------------------
415
416// The full landing reaches offline-landed with every landing item OK, the
417// row held, nothing served and nothing taking the target's lease; and the
418// lesson of the 2026-10-02 outage: the target's <host>.env, and so its
419// instance key, is the source's byte for byte, and its rendered
420// <host>.config.env carries the app's store binding, its secret included.
421func TestOfflineLandHappyPath(t *testing.T) {
422 lw := newLandWorld(t)
423 lw.wd.flip()
424 lw.uk.reconcile()
425 if rt := lw.uk.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
426 t.Fatalf("the target row before the landing: %+v", rt)
427 }
428 lw.uk.edges()
429 lw.uk.pass()
430 lw.requireLanded()
431 lw.requireSourceBytes()
432
433 // The instance key the app would be handed, byte for byte.
434 read := func(b *offBox, name string) map[string]string {
435 body, err := os.ReadFile(filepath.Join(b.dd, name))
436 if err != nil {
437 t.Fatal(err)
438 }
439 return instancekey.ParseEnv(body)
440 }
441 srcKey, _ := instancekey.Effective(read(lw.ie, offHost+".config.env"), read(lw.ie, offHost+".env"))
442 dstKey, layer := instancekey.Effective(read(lw.uk, offHost+".config.env"), read(lw.uk, offHost+".env"))
443 if srcKey != landKeyValue || dstKey != srcKey || layer != instancekey.LayerOwn {
444 t.Fatalf("instance key: source %q, target %q (layer %v)", srcKey, dstKey, layer)
445 }
446 // The store binding, as the target will launch it.
447 env, _, _, err := lw.ukEB.LaunchLayers(*lw.uk.row(offHost))
448 if err != nil {
449 t.Fatal(err)
450 }
451 for k, want := range map[string]string{"STORE_BUCKET": landStore, "STORE_ENDPOINT": "https://store.example.com", "STORE_SECRET_KEY": landSecret, "CARLOS_INSTANCE_KEY": landKeyValue, "CARLOS_ADMIN_TOKEN": "token"} {
452 if env[k] != want {
453 t.Fatalf("the target launches with %s=%q, want %q", k, env[k], want)
454 }
455 }
456 // The ack names the move; the staged replica is untouched.
457 ack := appconfig.ConfigAck{}
458 body, _, err := lw.wd.s.Get(context.Background(), appconfig.ConfigAckKey("uk-box", offHost))
459 if err != nil || json.Unmarshal(body, &ack) != nil || ack.MoveOp != offID {
460 t.Fatalf("config ack %+v %v", ack, err)
461 }
462 staged := lw.s3.current(t, "data-uk", activator.MovesPrefix(offHost))
463 if len(staged) != len(lw.seal.Staged) {
464 t.Fatalf("staged %v, sealed %v", keysOf(staged), keysOf(lw.seal.Staged))
465 }
466 // Timers, NextDue and replication arrived.
467 rows, err := lw.uk.reg.TimersFor(offHost)
468 if err != nil || moves.TimersDigest(rows) != lw.seal.TimersDigest || len(rows) != 2 {
469 t.Fatalf("timers %+v %v", rows, err)
470 }
471 if rt := lw.uk.row(offHost); rt.NextDue != 1_800_000_000 || rt.Replication != registry.ReplicationBatched {
472 t.Fatalf("row columns %+v", rt)
473 }
474 if lw.parked(moves.OfflineParkTargetBefore) != nil || lw.parked(moves.OfflineParkVerify) != nil {
475 t.Fatal("a clean landing parked something or left its verify scratch")
476 }
477}
478
479// --- (c) B2: every pass is an ensure ---------------------------------------------
480
481// A second, third and post-CAS pass, and a pass after an agent restart,
482// change nothing on disk, write nothing to the staged replica or the
483// archive, and report the same items.
484func TestOfflineLandIsAnEnsure(t *testing.T) {
485 lw := newLandWorld(t)
486 lw.flipAndLand(1)
487 lw.requireLanded()
488 disk := tree(t, lw.uk.dd)
489 items := itemsSummary(lw.report())
490 data := lw.s3.current(t, "data-uk", "moves/")
491 for i := range 4 {
492 if i == 2 {
493 lw.restart()
494 }
495 lw.uk.tick()
496 lw.requireLanded()
497 if got := tree(t, lw.uk.dd); !reflect.DeepEqual(got, disk) {
498 t.Fatalf("pass %d changed the disk:\nbefore %v\nafter %v", i, disk, got)
499 }
500 if got := itemsSummary(lw.report()); !reflect.DeepEqual(got, items) {
501 t.Fatalf("pass %d reports %+v, the first %+v", i, got, items)
502 }
503 if got := lw.s3.current(t, "data-uk", "moves/"); !reflect.DeepEqual(got, data) {
504 t.Fatalf("pass %d changed data-uk moves/", i)
505 }
506 }
507}
508
509// A crash at every point of the landing (after each item, after the report,
510// after the CAS), followed by an agent restart, converges: the next passes
511// land everything, park nothing, and leave the disk as a clean landing does.
512func TestOfflineLandConvergesAfterACrashAtEveryPoint(t *testing.T) {
513 var points []string
514 for _, id := range []string{"warm-cache", "key-adoption", "lease", "db", "files", "config", "env", "env-pin", "timers", "next-due", "replication", "release", "report", "landed"} {
515 points = append(points, "offline-"+id)
516 }
517 for _, point := range points {
518 t.Run(point, func(t *testing.T) {
519 lw := newLandWorld(t)
520 crashed := 0
521 lw.uk.w.cfg.landFault = func(p string) error {
522 if p == point && crashed == 0 {
523 crashed++
524 return errors.New("crash")
525 }
526 return nil
527 }
528 lw.flipAndLand(1)
529 if crashed != 1 {
530 t.Fatalf("the crash point %s was never reached", point)
531 }
532 lw.restart()
533 lw.uk.tick()
534 lw.uk.tick()
535 lw.requireLanded()
536 lw.requireSourceBytes()
537 if p := lw.parked(moves.OfflineParkTargetBefore); p != nil {
538 t.Fatalf("parked its own landing: %v", p)
539 }
540 })
541 }
542}
543
544// --- (b) missing or corrupted items: mended -----------------------------------------
545
546// Each item the landing carries, removed or damaged after the landing, is
547// put back by the next pass: a missing one landed, a different one parked
548// into target-before and landed again. The report stays landed and the
549// record offline-landed.
550func TestOfflineLandMendsWhatIsMissing(t *testing.T) {
551 cases := []struct {
552 name string
553 damage func(lw *landWorld)
554 parked []string // names that must appear in target-before
555 }{
556 {"files directory removed", func(lw *landWorld) { os.RemoveAll(filepath.Join(lw.uk.dd, offHost)) }, nil},
557 {"a file changed", func(lw *landWorld) {
558 writeFile(lw.t, filepath.Join(lw.uk.dd, offHost, "instance.key"), "tampered\n")
559 }, []string{offHost + "/instance.key"}},
560 {"a file added", func(lw *landWorld) {
561 writeFile(lw.t, filepath.Join(lw.uk.dd, offHost, "stray"), "x")
562 }, []string{offHost + "/stray"}},
563 {"env removed", func(lw *landWorld) { os.Remove(filepath.Join(lw.uk.dd, offHost+".env")) }, []string{offHost + "/instance.key"}},
564 {"env changed: a new instance key", func(lw *landWorld) {
565 writeFile(lw.t, filepath.Join(lw.uk.dd, offHost+".env"), "CARLOS_INSTANCE_KEY=minted-on-the-target\n")
566 }, []string{offHost + ".env", offHost + "/instance.key"}},
567 {"config removed", func(lw *landWorld) { os.Remove(filepath.Join(lw.uk.dd, offHost+".config.env")) }, nil},
568 {"config changed", func(lw *landWorld) {
569 writeFile(lw.t, filepath.Join(lw.uk.dd, offHost+".config.env"), "STORE_BUCKET=elsewhere\n")
570 }, nil},
571 {"a timer removed", func(lw *landWorld) {
572 if err := lw.uk.reg.ReplaceTimers(offHost, nil); err != nil {
573 lw.t.Fatal(err)
574 }
575 }, nil},
576 {"next due changed", func(lw *landWorld) { lw.uk.reg.SetNextDue(offHost, 5) }, nil},
577 {"replication changed", func(lw *landWorld) { lw.uk.reg.SetReplication(offHost, registry.ReplicationDaily) }, nil},
578 {"a local database appeared", func(lw *landWorld) {
579 writeFile(lw.t, provision.DefaultDBPath(lw.uk.dd, offHost), "stale")
580 writeFile(lw.t, filepath.Join(activatorMetaDir(provision.DefaultDBPath(lw.uk.dd, offHost)), "ltx", "0", "x"), "meta")
581 }, []string{offHost + ".db", ".activator-" + offHost + ".db-litestream/ltx/0/x"}},
582 {"an adoption record appeared", func(lw *landWorld) {
583 writeFile(lw.t, filepath.Join(lw.uk.dd, offHost+instancekey.AdoptionSuffix), "{}")
584 }, []string{offHost + instancekey.AdoptionSuffix}},
585 }
586 for _, c := range cases {
587 t.Run(c.name, func(t *testing.T) {
588 lw := newLandWorld(t)
589 lw.flipAndLand(1)
590 lw.requireLanded()
591 c.damage(lw)
592 lw.uk.tick()
593 lw.requireLanded()
594 lw.requireSourceBytes()
595 if rows, _ := lw.uk.reg.TimersFor(offHost); moves.TimersDigest(rows) != lw.seal.TimersDigest {
596 t.Fatal("timers not mended")
597 }
598 if rt := lw.uk.row(offHost); rt.NextDue != lw.seal.NextDue || rt.Replication != lw.seal.Replication {
599 t.Fatalf("row columns not mended: %+v", rt)
600 }
601 parked := lw.parked(moves.OfflineParkTargetBefore)
602 for _, name := range c.parked {
603 if _, ok := parked[name]; !ok {
604 t.Fatalf("%s not parked; target-before holds %v", name, keysOf(parked))
605 }
606 }
607 if len(c.parked) == 0 && parked != nil {
608 t.Fatalf("parked %v", keysOf(parked))
609 }
610 // And it stays mended: the next pass parks nothing more.
611 before := tree(t, lw.uk.dd)
612 lw.uk.tick()
613 if got := tree(t, lw.uk.dd); !reflect.DeepEqual(got, before) {
614 t.Fatal("the pass after the mend changed the disk")
615 }
616 })
617 }
618}
619
620// What a target holds for the host before the landing (an old residence's
621// database, meta dir, adoption record, files and env) is parked, never
622// trusted; a stale tombstone of another move in the target's bucket is
623// lifted; and the landing completes.
624func TestOfflineLandParksAndLiftsWhatItFinds(t *testing.T) {
625 lw := newLandWorld(t)
626 ctx := context.Background()
627 dd := lw.uk.dd
628 db := provision.DefaultDBPath(dd, offHost)
629 writeFile(t, db, "old residence")
630 writeFile(t, db+"-wal", "wal")
631 writeFile(t, filepath.Join(activatorMetaDir(db), "ltx", "0", "0000000000000009-0000000000000009.ltx"), "meta")
632 writeFile(t, filepath.Join(dd, offHost+instancekey.AdoptionSuffix), "{}")
633 writeFile(t, filepath.Join(dd, offHost, "old.txt"), "old residence's file")
634 writeFile(t, filepath.Join(dd, offHost+".env"), "CARLOS_INSTANCE_KEY=old-residence-key\n")
635 loc := activator.HibernatePrefix("instances", offHost)
636 if err := lease.Tombstone(ctx, "data-uk", loc, "eu-west-2", moves.TombstoneOwner(offID2), time.Hour); err != nil {
637 t.Fatal(err)
638 }
639 lw.flipAndLand(1)
640 lw.requireLanded()
641 lw.requireSourceBytes()
642 parked := lw.parked(moves.OfflineParkTargetBefore)
643 for _, name := range []string{offHost + ".db", offHost + ".db-wal", ".activator-" + offHost + ".db-litestream/ltx/0/0000000000000009-0000000000000009.ltx",
644 offHost + instancekey.AdoptionSuffix, offHost + "/old.txt", offHost + ".env"} {
645 if _, ok := parked[name]; !ok {
646 t.Fatalf("%s not parked: %v", name, keysOf(parked))
647 }
648 }
649 _, owner, expires, err := lease.LockState(ctx, "data-uk", loc, "eu-west-2")
650 if err != nil || owner != moves.TombstoneOwner(offID2) || !lease.IsLiftedTombstone(owner, expires) {
651 t.Fatalf("the stale tombstone: %q %v %v", owner, expires, err)
652 }
653 // Lifted, not taken: Acquire would now succeed, and nothing else
654 // appeared under instances/.
655 if got := keysOf(lw.s3.current(t, "data-uk", "instances/")); len(got) != 1 || got[0] != loc+"/lock.json" {
656 t.Fatalf("data-uk instances/: %v", got)
657 }
658}
659
660// --- (b) mismatches: failed, named --------------------------------------------------
661
662// Every mismatch with the seal fails its own item in a failed report, and
663// the record stays offline-landing: an extra staged object, a changed one,
664// a changed archive, a changed bundle, a promoted channel, a seal that is
665// not the record's, a changed timers snapshot, a live writer's lease, and a
666// tombstone of this very move in the target's bucket.
667func TestOfflineLandFailsEachMismatch(t *testing.T) {
668 prefix := moves.OfflineArchivePrefix(offHost, offID)
669 cases := []struct {
670 name string
671 item string
672 hard bool
673 damage func(lw *landWorld)
674 }{
675 {"an extra staged object", "db", true, func(lw *landWorld) {
676 lw.s3.f.Put("data-uk", activator.MovesPrefix(offHost)+"0000/0000000000000009-0000000000000009.ltx", []byte("x"))
677 }},
678 {"a changed staged object", "db", true, func(lw *landWorld) {
679 k := keysOf(lw.seal.Staged)[0]
680 lw.s3.f.Put("data-uk", activator.MovesPrefix(offHost)+k, []byte("not the sealed bytes"))
681 }},
682 {"a staged object gone", "db", true, func(lw *landWorld) {
683 lw.s3.f.BareDelete("data-uk", activator.MovesPrefix(offHost)+keysOf(lw.seal.Staged)[0])
684 }},
685 {"a changed archive", "files", true, func(lw *landWorld) {
686 b, _ := lw.s3.f.Current("data-uk", prefix+"files.sealed")
687 b = append([]byte(nil), b...)
688 b[len(b)/2] ^= 0xff
689 lw.s3.f.Put("data-uk", prefix+"files.sealed", b)
690 }},
691 {"a changed bundle", "env-pin", true, func(lw *landWorld) { lw.bundle("shop-store-somewhere-else") }},
692 {"a promoted channel", "release", true, func(lw *landWorld) { lw.promote("v2") }},
693 {"a changed timers snapshot", "timers", true, func(lw *landWorld) {
694 snap, _ := moves.TimersSnapshot{Host: offHost, TakenAt: lw.wd.clock()}.Encode()
695 lw.s3.f.Put("data-uk", prefix+"timers.json", snap)
696 }},
697 {"a live writer holds the target's lease", "lease", false, func(lw *landWorld) {
698 held, err := lease.Acquire(context.Background(), "data-uk", activator.HibernatePrefix("instances", offHost), "eu-west-2")
699 if err != nil {
700 lw.t.Fatal(err)
701 }
702 lw.t.Cleanup(func() { held.Release(context.Background()) })
703 }},
704 {"this move's own tombstone in the target's bucket", "lease", true, func(lw *landWorld) {
705 if err := lease.Tombstone(context.Background(), "data-uk", activator.HibernatePrefix("instances", offHost), "eu-west-2", moves.TombstoneOwner(offID), time.Hour); err != nil {
706 lw.t.Fatal(err)
707 }
708 }},
709 }
710 for _, c := range cases {
711 t.Run(c.name, func(t *testing.T) {
712 lw := newLandWorld(t)
713 c.damage(lw)
714 lw.flipAndLand(2)
715 if st := lw.state(); st != provision.OfflineLanding {
716 t.Fatalf("state %s", st)
717 }
718 r := lw.report()
719 if r.Items[c.item].OK || r.Items[c.item].Reason == "" {
720 t.Fatalf("%s: %+v", c.item, r.Items[c.item])
721 }
722 if want := map[bool]string{true: moves.OfflinePhaseFailed, false: moves.OfflinePhaseLanded}[c.hard]; r.Phase != want {
723 t.Fatalf("phase %s, want %s (%+v)", r.Phase, want, r.Items)
724 }
725 if rt := lw.uk.row(offHost); rt == nil || rt.HoldOp != offID {
726 t.Fatalf("row %+v", rt)
727 }
728 lw.uk.edges()
729 if lw.uk.lists(offHost) {
730 t.Fatal("listed")
731 }
732 })
733 }
734
735 // A seal whose bytes are not the ones the record names fails every item
736 // after the row; nothing is landed.
737 t.Run("a seal that is not the record's", func(t *testing.T) {
738 lw := newLandWorld(t)
739 if err := moves.PutReplace(context.Background(), lw.wd.s, moves.OfflineSealKey(offHost, offID), append(lw.sealRaw, ' ')); err != nil {
740 t.Fatal(err)
741 }
742 lw.flipAndLand(1)
743 r := lw.report()
744 if r.Phase != moves.OfflinePhaseFailed || !r.Items["row"].OK {
745 t.Fatalf("%+v", r)
746 }
747 for _, it := range moves.OfflineOwned(moves.OfflineOwnerLanding) {
748 if it.ID != "row" && r.Items[it.ID].OK {
749 t.Fatalf("%s OK under a seal that is not the record's", it.ID)
750 }
751 }
752 if isDir(filepath.Join(lw.uk.dd, offHost)) || lexists(filepath.Join(lw.uk.dd, offHost+".env")) {
753 t.Fatal("landed files under a seal that is not the record's")
754 }
755 })
756
757 // A mismatch that clears (the channel promoted back) lands on a later
758 // pass: the report is recomputed from what is there every pass.
759 t.Run("a mismatch that clears", func(t *testing.T) {
760 lw := newLandWorld(t)
761 lw.promote("v2")
762 lw.flipAndLand(1)
763 if r := lw.report(); r.Phase != moves.OfflinePhaseFailed || r.Items["release"].OK {
764 t.Fatalf("%+v", r.Items["release"])
765 }
766 lw.promote(sealVersion)
767 lw.uk.tick()
768 lw.requireLanded()
769 })
770}
771
772// --- (d) open ---------------------------------------------------------------------
773
774// open releases the hold only once every landing item verifies again
775// against the seal: a promote after the landing keeps the row held; once
776// it is undone the hold is released, the row is ready and listed, the
777// archive is deleted from the target's bucket, and the report says opened.
778// A following ExecBackend.Restore promotes the staged replica and its
779// database passes CheckDB with every row; until that first wake nothing has
780// taken the target's lease.
781func TestOfflineLandOpenReleasesOnlyWhenEveryItemVerifies(t *testing.T) {
782 lw := newLandWorld(t)
783 ctx := context.Background()
784 // A fresh target: no live link for the host, which the release item
785 // accepts (nothing can run until adoption stages the sealed build).
786 if lexists(release.LiveLink(lw.uk.opt, offHost)) {
787 t.Fatal("fixture: the target already has a live link")
788 }
789 lw.flipAndLand(1)
790 lw.requireLanded()
791 lw.wd.open()
792
793 // An item that no longer verifies: not opened.
794 lw.promote("v2")
795 lw.uk.tick()
796 if rt := lw.uk.row(offHost); rt == nil || rt.HoldOp != offID || rt.Status != "landing" {
797 t.Fatalf("released with the release item failing: %+v", rt)
798 }
799 if r := lw.report(); r.Phase == moves.OfflinePhaseOpened || r.Items["release"].OK {
800 t.Fatalf("report at open: %s %+v", r.Phase, r.Items["release"])
801 }
802 lw.requireNoTargetLease()
803
804 // An item the open's ensure mends (a timer lost) does not block it.
805 lw.promote(sealVersion)
806 if err := lw.uk.reg.ReplaceTimers(offHost, nil); err != nil {
807 t.Fatal(err)
808 }
809 lw.uk.tick()
810 rt := lw.uk.row(offHost)
811 if rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
812 t.Fatalf("not opened: %+v (%+v)", rt, lw.report().Items)
813 }
814 if rows, _ := lw.uk.reg.TimersFor(offHost); moves.TimersDigest(rows) != lw.seal.TimersDigest {
815 t.Fatal("timers not mended before the release")
816 }
817 r := lw.report()
818 if r.Phase != moves.OfflinePhaseOpened {
819 t.Fatalf("phase %s", r.Phase)
820 }
821 if ok, bad := r.ItemsOK(moves.OfflineOwnerLanding); !ok {
822 t.Fatalf("opened report: %v", bad)
823 }
824 lw.uk.edges()
825 if !lw.uk.lists(offHost) {
826 t.Fatal("the edges file does not list the opened host")
827 }
828 if got := lw.s3.current(t, "data-uk", moves.OfflineArchivePrefix(offHost, offID)); len(got) != 0 {
829 t.Fatalf("the archive is still there: %v", keysOf(got))
830 }
831 lw.requireNoTargetLease()
832 lw.requireSourceBytes()
833
834 // The first wake's restore: staging promoted into data-uk's own prefix,
835 // the database restored and checked.
836 m, err := lease.NewS3Mirror(ctx, "eu-west-2", "eu-west-2")
837 if err != nil {
838 t.Fatal(err)
839 }
840 eb := &activator.ExecBackend{Cfg: activator.Config{Bucket: "data-uk", Region: "eu-west-2", Prefix: "instances"}, DataDir: lw.uk.dd, OptDir: lw.uk.opt, Mirror: m}
841 fresh, err := eb.Restore(ctx, *rt)
842 if err != nil || fresh {
843 t.Fatalf("restore: fresh %v, %v", fresh, err)
844 }
845 if err := lease.CheckDB(rt.DBPath); err != nil {
846 t.Fatal(err)
847 }
848 if n := countRows(t, rt.DBPath); n != 20 {
849 t.Fatalf("restored %d rows, want 20", n)
850 }
851 if got := lw.s3.current(t, "data-uk", activator.MovesPrefix(offHost)); len(got) != 0 {
852 t.Fatalf("staging left after the promotion: %v", keysOf(got))
853 }
854
855 // Later passes at open keep the report and do not re-verify the
856 // staging the wake consumed.
857 lw.uk.tick()
858 lw.restart()
859 lw.uk.tick()
860 if r := lw.report(); r.Phase != moves.OfflinePhaseOpened {
861 t.Fatalf("phase after the wake: %s", r.Phase)
862 }
863 if rt := lw.uk.row(offHost); rt.Status != "ready" {
864 t.Fatalf("row %+v", rt)
865 }
866}
867
868// --- (e) the target's revert ----------------------------------------------------------
869
870// An abort after the flip parks what the target landed (files, env, config)
871// under the move's target park directory, deletes the held row and its
872// timers, reports reverted, and leaves data-uk instances/<host>/ untouched;
873// a later pass changes nothing.
874func TestOfflineLandTargetRevert(t *testing.T) {
875 lw := newLandWorld(t)
876 lw.flipAndLand(1)
877 lw.requireLanded()
878 landed := tree(t, filepath.Join(lw.uk.dd, offHost))
879 lw.wd.abort()
880 lw.uk.tick()
881 if rt := lw.uk.row(offHost); rt != nil {
882 t.Fatalf("the row is still there: %+v", rt)
883 }
884 if rows, _ := lw.uk.reg.TimersFor(offHost); len(rows) != 0 {
885 t.Fatalf("timers left: %+v", rows)
886 }
887 for _, name := range []string{offHost, offHost + ".env", offHost + ".config.env"} {
888 if lexists(filepath.Join(lw.uk.dd, name)) {
889 t.Fatalf("%s left in place", name)
890 }
891 }
892 parkDir := moves.OfflineParkDir(lw.uk.dd, offHost, offID, moves.OfflineRoleTarget)
893 if got := tree(t, filepath.Join(parkDir, offHost)); !reflect.DeepEqual(got, landed) {
894 t.Fatalf("parked files %v, landed %v", got, landed)
895 }
896 for _, name := range []string{offHost + ".env", offHost + ".config.env"} {
897 if !lexists(filepath.Join(parkDir, name)) {
898 t.Fatalf("%s not parked", name)
899 }
900 }
901 if r := lw.report(); r.Phase != moves.OfflinePhaseReverted || r.RowPresent {
902 t.Fatalf("report %+v", r)
903 }
904 lw.requireNoTargetLease()
905 if lw.state() != provision.OfflineReverting {
906 t.Fatalf("state %s", lw.state())
907 }
908 disk := tree(t, lw.uk.dd)
909 lw.restart()
910 lw.uk.tick()
911 lw.uk.tick()
912 if rt := lw.uk.row(offHost); rt != nil {
913 t.Fatalf("the row came back: %+v", rt)
914 }
915 if got := tree(t, lw.uk.dd); !reflect.DeepEqual(got, disk) {
916 t.Fatal("a later revert pass changed the disk")
917 }
918 lw.uk.edges()
919 if lw.uk.lists(offHost) {
920 t.Fatal("listed after the revert")
921 }
922}
923
924// An abort after the flip on a target that never landed: the held row
925// reconcile minted is deleted, nothing is parked, and a target that never
926// had a row does nothing at all.
927func TestOfflineLandTargetRevertBeforeTheLanding(t *testing.T) {
928 lw := newLandWorld(t)
929 // The target mints its row held and runs its moves pass, which crashes
930 // before anything is landed or reported.
931 lw.uk.w.cfg.landFault = func(p string) error {
932 if p == "offline-warm-cache" {
933 return errors.New("crash")
934 }
935 return nil
936 }
937 lw.flipAndLand(1)
938 if rt := lw.uk.row(offHost); rt == nil || rt.HoldOp != offID {
939 t.Fatalf("row %+v", rt)
940 }
941 if _, ok := lw.wd.reportIf(moves.OfflineRoleTarget, "uk-box"); ok {
942 t.Fatal("reported before the crash point")
943 }
944 lw.wd.abort()
945 lw.restart()
946 lw.uk.tick()
947 if rt := lw.uk.row(offHost); rt != nil {
948 t.Fatalf("row %+v", rt)
949 }
950 if isDir(moves.OfflineParkDir(lw.uk.dd, offHost, offID, moves.OfflineRoleTarget)) {
951 t.Fatal("parked something it never landed")
952 }
953 if r := lw.report(); r.Phase != moves.OfflinePhaseReverted {
954 t.Fatalf("report %+v", r)
955 }
956
957 lw2 := newLandWorld(t)
958 lw2.wd.flip()
959 lw2.wd.abort()
960 lw2.uk.tick()
961 if rt := lw2.uk.row(offHost); rt != nil {
962 t.Fatalf("row %+v", rt)
963 }
964 if _, ok := lw2.wd.reportIf(moves.OfflineRoleTarget, "uk-box"); ok {
965 t.Fatal("a target that never had a row reported")
966 }
967}
968
969// The instance key the app is handed may come from the app's configuration
970// rather than <host>.env (instancekey.Effective). The env item judges the key
971// the target would hand over, from both files as the activator merges them:
972// a configured key lands; a configured key changed after the seal fails the
973// env item by name (and the env pin), so the target never hands the app a key
974// that is not the source's.
975func TestOfflineLandConfiguredInstanceKey(t *testing.T) {
976 lw := newLandWorldWith(t, landOpts{configKey: landKeyValue})
977 lw.flipAndLand(1)
978 lw.requireLanded()
979 lw.requireSourceBytes()
980 env, _, _, err := lw.ukEB.LaunchLayers(*lw.uk.row(offHost))
981 if err != nil || env[instancekey.Var] != landKeyValue {
982 t.Fatalf("the target hands over %q (%v)", env[instancekey.Var], err)
983 }
984
985 lw2 := newLandWorldWith(t, landOpts{configKey: landKeyValue})
986 lw2.opts.configKey = "ik_ffffffffffffffffffffffffffffffff"
987 lw2.bundle(landStore)
988 lw2.flipAndLand(2)
989 if st := lw2.state(); st != provision.OfflineLanding {
990 t.Fatalf("state %s", st)
991 }
992 r := lw2.report()
993 if c := r.Items["env"]; c.OK || !strings.Contains(c.Reason, "instance key") || r.Phase != moves.OfflinePhaseFailed {
994 t.Fatalf("env %+v, phase %s", c, r.Phase)
995 }
996 if r.Items["env-pin"].OK {
997 t.Fatal("the env pin passed with another key")
998 }
999}
1000
1001// Review I1: a first landing pass that extracted the files and env and
1002// rendered the config, then crashed before it could report, followed by an
1003// abort: the revert parks all three while the row is still held, before it
1004// deletes the row, so nothing of the move's is left in the data directory.
1005func TestOfflineLandRevertParksAfterACrashBeforeTheReport(t *testing.T) {
1006 lw := newLandWorld(t)
1007 lw.crashAt("offline-release")
1008 lw.flipAndLand(1)
1009 if _, ok := lw.wd.reportIf(moves.OfflineRoleTarget, "uk-box"); ok {
1010 t.Fatal("reported before the crash point")
1011 }
1012 names := []string{offHost, offHost + ".env", offHost + ".config.env"}
1013 for _, name := range names {
1014 if !lexists(filepath.Join(lw.uk.dd, name)) {
1015 t.Fatalf("fixture: %s was not landed before the crash", name)
1016 }
1017 }
1018 landed := tree(t, filepath.Join(lw.uk.dd, offHost))
1019 lw.wd.abort()
1020 lw.restart()
1021 lw.uk.tick()
1022 parkDir := moves.OfflineParkDir(lw.uk.dd, offHost, offID, moves.OfflineRoleTarget)
1023 for _, name := range names {
1024 if lexists(filepath.Join(lw.uk.dd, name)) {
1025 t.Fatalf("%s left in the data directory", name)
1026 }
1027 if !lexists(filepath.Join(parkDir, name)) {
1028 t.Fatalf("%s not parked", name)
1029 }
1030 }
1031 if got := tree(t, filepath.Join(parkDir, offHost)); !reflect.DeepEqual(got, landed) {
1032 t.Fatalf("parked %v, landed %v", got, landed)
1033 }
1034 if rt := lw.uk.row(offHost); rt != nil {
1035 t.Fatalf("row %+v", rt)
1036 }
1037 if r := lw.report(); r.Phase != moves.OfflinePhaseReverted {
1038 t.Fatalf("report %+v", r)
1039 }
1040}
1041
1042// Review I2: a failed env pin names the variables that differ, from the
1043// source's env report beside the archive, and never their values.
1044func TestOfflineLandEnvPinNamesTheVariables(t *testing.T) {
1045 lw := newLandWorld(t)
1046 lw.bundle("shop-store-somewhere-else")
1047 lw.flipAndLand(1)
1048 c := lw.report().Items["env-pin"]
1049 if c.OK || !strings.Contains(c.Reason, "STORE_BUCKET") || strings.Contains(c.Reason, "somewhere-else") || strings.Contains(c.Reason, landStore) {
1050 t.Fatalf("env-pin %+v", c)
1051 }
1052 for _, name := range []string{"STORE_ENDPOINT", "STORE_SECRET_KEY", "FEATURE", "CARLOS_REGION"} {
1053 if strings.Contains(c.Reason, name) {
1054 t.Fatalf("env-pin names %s, which did not change: %q", name, c.Reason)
1055 }
1056 }
1057 // Without the source's report the pin still fails, by digest alone.
1058 lw2 := newLandWorld(t)
1059 lw2.bundle("shop-store-somewhere-else")
1060 lw2.s3.f.BareDelete(sealUK, moves.OfflineArchivePrefix(offHost, offID)+offlineEnvReportObject)
1061 lw2.flipAndLand(1)
1062 if c := lw2.report().Items["env-pin"]; c.OK || c.Reason != "the launch environment differs from the source's" {
1063 t.Fatalf("env-pin without the source's report %+v", c)
1064 }
1065}
1066
1067// Review M1: the verified report is written before the release, so a crash
1068// after the release leaves the items that pass verified (here after an
1069// earlier open pass failed), and the next pass reports opened with every
1070// item OK.
1071func TestOfflineLandOpenAfterACrashPastTheRelease(t *testing.T) {
1072 lw := newLandWorld(t)
1073 lw.flipAndLand(1)
1074 lw.requireLanded()
1075 lw.wd.open()
1076 lw.promote("v2")
1077 lw.uk.tick()
1078 if r := lw.report(); r.Items["release"].OK {
1079 t.Fatal("fixture: the release item passed")
1080 }
1081 lw.promote(sealVersion)
1082 crashed := lw.crashAt("offline-released")
1083 lw.uk.tick()
1084 if *crashed != 1 {
1085 t.Fatal("the crash point was never reached")
1086 }
1087 if rt := lw.uk.row(offHost); rt == nil || rt.Status != "ready" {
1088 t.Fatalf("row %+v", rt)
1089 }
1090 if r := lw.report(); r.Phase == moves.OfflinePhaseOpened {
1091 t.Fatal("fixture: reported opened before the crash")
1092 } else if ok, bad := r.ItemsOK(moves.OfflineOwnerLanding); !ok {
1093 t.Fatalf("the report before the release carries %v not OK", bad)
1094 }
1095 lw.restart()
1096 lw.uk.tick()
1097 r := lw.report()
1098 if ok, bad := r.ItemsOK(moves.OfflineOwnerLanding); r.Phase != moves.OfflinePhaseOpened || !ok {
1099 t.Fatalf("phase %s, not OK %v", r.Phase, bad)
1100 }
1101 if got := lw.s3.current(t, sealUK, moves.OfflineArchivePrefix(offHost, offID)); len(got) != 0 {
1102 t.Fatalf("the archive is still there: %v", keysOf(got))
1103 }
1104}
1105
1106// T4's contract for a host with no files directory: the archive carries no
1107// files/ entry and requires no file. The landing accepts it: extraction
1108// leaves an empty <DataDir>/<host> (0755), whose digest with the env file is
1109// the seal's, and later passes leave it alone.
1110func TestOfflineLandAHostWithNoFilesDirectory(t *testing.T) {
1111 lw := newLandWorldWith(t, landOpts{noFiles: true})
1112 if lw.seal.FilesEntries != 1 {
1113 t.Fatalf("fixture: the seal carries %d entries, want the env alone", lw.seal.FilesEntries)
1114 }
1115 lw.flipAndLand(1)
1116 lw.requireLanded()
1117 lw.requireSourceBytes()
1118 st, err := os.Stat(filepath.Join(lw.uk.dd, offHost))
1119 if err != nil || st.Mode().Perm() != 0o755 {
1120 t.Fatalf("the files directory: %v %v", st, err)
1121 }
1122 disk := tree(t, lw.uk.dd)
1123 lw.uk.tick()
1124 lw.requireLanded()
1125 if got := tree(t, lw.uk.dd); !reflect.DeepEqual(got, disk) {
1126 t.Fatal("a later pass changed the disk")
1127 }
1128}
1129
1130// A nested source keeps launch descriptors (launch.<role>.json and
1131// launch.gen.<n>.json) beside its files directory. They are the platform's,
1132// not carried, and the landing neither expects nor makes them: it lands the
1133// files directory alone, flat.
1134func TestOfflineLandANestedSourceWithLaunchFiles(t *testing.T) {
1135 lw := newLandWorldWith(t, landOpts{nested: true})
1136 lw.flipAndLand(1)
1137 lw.requireLanded()
1138 lw.requireSourceBytes()
1139 for rel := range tree(t, lw.uk.dd) {
1140 if strings.Contains(filepath.Base(rel), "launch.") {
1141 t.Fatalf("the target holds %s", rel)
1142 }
1143 }
1144}
1145
1146// --- the target's own live build (final review, return moves) --------------------
1147
1148const landOlderBuild = "the target box still holds an older build of this instance. see the offline move runbook"
1149
1150// stageLive gives the target box a live link for the host naming version v
1151// of the app, as a box the instance once left keeps it after retirement: a
1152// staged executable in the version store and the link to it.
1153func (lw *landWorld) stageLive(v string) string {
1154 t := lw.t
1155 t.Helper()
1156 writeFile(t, release.VersionPath(lw.uk.opt, lw.wd.acct, v, "app"), "#!/bin/sh\nexit 0\n")
1157 if err := release.PointLive(lw.uk.opt, offHost, lw.wd.acct, v, "app"); err != nil {
1158 t.Fatal(err)
1159 }
1160 target, err := os.Readlink(release.LiveLink(lw.uk.opt, offHost))
1161 if err != nil {
1162 t.Fatal(err)
1163 }
1164 return target
1165}
1166
1167// requireNoWake: the row is still held landing by the move, the edges file
1168// does not list it (so the edge never hands it to the activator), and a
1169// scheduled tick for it, the one wake source that reaches a held row, is
1170// answered with the maintenance page without reaching the exec backend.
1171func (lw *landWorld) requireNoWake(when string) {
1172 t := lw.t
1173 t.Helper()
1174 rt := lw.uk.row(offHost)
1175 if rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
1176 t.Fatalf("%s: the target row %+v is not held by the move", when, rt)
1177 }
1178 lw.uk.edges()
1179 if lw.uk.lists(offHost) {
1180 t.Fatalf("%s: the edges file lists the host", when)
1181 }
1182 proxied := false
1183 rec := httptest.NewRecorder()
1184 lw.uk.svc.ServeScheduled(rec, httptest.NewRequest("POST", "http://"+offHost+"/jobs/remind", nil), *rt,
1185 http.HandlerFunc(func(http.ResponseWriter, *http.Request) { proxied = true }))
1186 if proxied || rec.Code != http.StatusServiceUnavailable {
1187 t.Fatalf("%s: a scheduled tick on the target was answered %d (proxied %v)", when, rec.Code, proxied)
1188 }
1189 if rt := lw.uk.row(offHost); rt == nil || rt.Status != "landing" || rt.HoldOp != offID {
1190 t.Fatalf("%s: after the tick the target row is %+v", when, rt)
1191 }
1192}
1193
1194// A return move after an upgrade: the target box is one the instance left
1195// on an older build, and it still keeps that build's live link (retirement
1196// leaves the root-owned link). The channel names the sealed build, so only
1197// the link can tell. The landing fails the release item with MV-O177 and
1198// every other item lands; the record stays offline-landing; nothing can
1199// wake the row; the link is left exactly as it was. Open, reached even so,
1200// verifies the item again and keeps the hold. A link that names the sealed
1201// build passes, and open then releases the row.
1202func TestOfflineLandRefusesAnOlderLiveBuildOnTheTarget(t *testing.T) {
1203 lw := newLandWorld(t)
1204 old := lw.stageLive("v0")
1205 lw.flipAndLand(2)
1206 if st := lw.state(); st != provision.OfflineLanding {
1207 t.Fatalf("state %s with an older live build on the target", st)
1208 }
1209 r := lw.report()
1210 if r.Phase != moves.OfflinePhaseFailed || r.Items["release"].OK || r.Items["release"].Reason != landOlderBuild {
1211 t.Fatalf("phase %s, release %+v", r.Phase, r.Items["release"])
1212 }
1213 for id, c := range r.Items {
1214 if id != "release" && !c.OK {
1215 t.Fatalf("%s not OK: %+v", id, c)
1216 }
1217 }
1218 lw.requireNoWake("after the landing")
1219 if got, err := os.Readlink(release.LiveLink(lw.uk.opt, offHost)); err != nil || got != old {
1220 t.Fatalf("the live link is %q (%v), want it left at %q", got, err, old)
1221 }
1222
1223 // Open, forced past the door's landed gate, verifies again and refuses.
1224 lw.wd.landed()
1225 lw.wd.open()
1226 lw.uk.tick()
1227 lw.uk.tick()
1228 if r := lw.report(); r.Phase == moves.OfflinePhaseOpened || r.Items["release"].Reason != landOlderBuild {
1229 t.Fatalf("open with an older live build: %s %+v", r.Phase, r.Items["release"])
1230 }
1231 lw.requireNoWake("at open")
1232 lw.requireNoTargetLease()
1233
1234 // The same link naming the sealed build: open releases the row.
1235 lw.stageLive(sealVersion)
1236 lw.uk.tick()
1237 if rt := lw.uk.row(offHost); rt == nil || rt.Status != "ready" || rt.HoldOp != "" {
1238 t.Fatalf("not opened with the sealed live build: %+v (%+v)", rt, lw.report().Items)
1239 }
1240 if r := lw.report(); r.Phase != moves.OfflinePhaseOpened {
1241 t.Fatalf("phase %s", r.Phase)
1242 }
1243}
1244
1245// A live link that names another build fails at open too: the landing was
1246// clean (no link), and the link appears before open.
1247func TestOfflineLandOpenRefusesALiveBuildThatAppeared(t *testing.T) {
1248 lw := newLandWorld(t)
1249 lw.flipAndLand(1)
1250 lw.requireLanded()
1251 lw.stageLive("v0")
1252 lw.wd.open()
1253 lw.uk.tick()
1254 if r := lw.report(); r.Phase == moves.OfflinePhaseOpened || r.Items["release"].Reason != landOlderBuild {
1255 t.Fatalf("open with another live build: %s %+v", r.Phase, r.Items["release"])
1256 }
1257 lw.requireNoWake("at open")
1258}
1259