| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "os/exec" |
| 8 | "path/filepath" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/hostbox" |
| 13 | "github.com/carlosframework/platform/internal/hostclaim" |
| 14 | "github.com/carlosframework/platform/internal/hostguard" |
| 15 | "github.com/carlosframework/platform/internal/provision" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | ) |
| 18 | |
| 19 | func plan14InventoryProcessRoot(t *testing.T, dir string) (ClaimRootHTTP, *rollbackDisk) { |
| 20 | t.Helper() |
| 21 | store := release.NewDirStore(filepath.Join(dir, "deployment")) |
| 22 | journal, err := hostbox.OpenLaunchJournal(filepath.Join(dir, "launches.db")) |
| 23 | if err != nil { |
| 24 | t.Fatal(err) |
| 25 | } |
| 26 | t.Cleanup(func() { journal.Close() }) |
| 27 | journals, err := hostbox.NewJournalService(journal) |
| 28 | if err != nil { |
| 29 | t.Fatal(err) |
| 30 | } |
| 31 | disk := &rollbackDisk{dir: filepath.Join(dir, "root")} |
| 32 | guard := hostguard.Guard{DataDir: filepath.Join(dir, "data"), LockDir: filepath.Join(dir, "locks"), Claims: hostguard.StoreReader{Store: store}, Root: hostguard.NewRootAuthority(disk)} |
| 33 | actor := hostbox.Actor{Box: "box-1", Enrollment: "en", Revision: 1, Binding: "1", Region: "ie"} |
| 34 | roots := map[string]string{"data": guard.DataDir, "opt": filepath.Join(dir, "opt"), "units": filepath.Join(dir, "units")} |
| 35 | reg := restartTestRegistry(t) |
| 36 | census := &hostbox.EmptyInstallationCensus{Inventory: journals.Inventory, Registry: reg, Launches: journal, Box: actor.Box, Enrollment: actor.Enrollment, DirectoryRevision: "1", Roots: roots} |
| 37 | return ClaimRootHTTP{Journals: journals, Launches: journal, Roots: roots, InventoryActor: func(context.Context) (hostbox.Actor, error) { return actor, nil }, Reconcile: hostbox.ReconcileAdapter{Registry: reg, Inventory: journals.Inventory, Census: census, Box: actor.Box, Enrollment: actor.Enrollment}, Stop: ClaimStopHTTP{Coordinator: ClaimStopCoordinator{Guard: guard, Evidence: store}}}, disk |
| 38 | } |
| 39 | func TestPlan14InventoryProcessCrash(t *testing.T) { |
| 40 | t.Parallel() |
| 41 | if phase := os.Getenv("PLAN14_INVENTORY_CRASH"); phase != "" { |
| 42 | root, disk := plan14InventoryProcessRoot(t, os.Getenv("PLAN14_INVENTORY_DIR")) |
| 43 | writes := 0 |
| 44 | disk.beforeWrite = func(string, []byte) { |
| 45 | if phase == "database" && writes == 1 { |
| 46 | os.Exit(73) |
| 47 | } |
| 48 | } |
| 49 | disk.afterWrite = func(string, []byte) { |
| 50 | writes++ |
| 51 | if phase == "intent" && writes == 1 || phase == "snapshot" && writes == 2 { |
| 52 | os.Exit(73) |
| 53 | } |
| 54 | } |
| 55 | obs, err := hostclaim.Read(t.Context(), root.Stop.Coordinator.Evidence, recoveryHost) |
| 56 | if err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | h := obs.Claim.Holder |
| 60 | _, etag, err := root.Stop.Coordinator.Evidence.Get(t.Context(), h.Key) |
| 61 | if err != nil { |
| 62 | t.Fatal(err) |
| 63 | } |
| 64 | _, client := hostbox.RemoteJournals(root.commitRootInventory) |
| 65 | _, err = client.WriteExpected(t.Context(), hostbox.StopInventory{Complete: true, Host: obs.Host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: "box-1", Enrollment: "en", DirectoryRevision: "1"}, hostbox.InventoryExpectation{RecordKey: h.Key, RecordETag: etag, Target: hostguard.FenceTarget{Host: obs.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Box: "box-1", Enrollment: "en", ClaimETag: obs.ETag, Op: obs.Claim.Op.ID, Phase: obs.Claim.Op.Phase}}) |
| 66 | t.Fatal("crash boundary not reached", phase, err) |
| 67 | } |
| 68 | for _, phase := range []string{"intent", "database", "snapshot"} { |
| 69 | t.Run(phase, func(t *testing.T) { |
| 70 | dir := t.TempDir() |
| 71 | for _, name := range []string{"root", "data", "opt", "units", "locks", "child"} { |
| 72 | if err := os.Mkdir(filepath.Join(dir, name), 0700); err != nil { |
| 73 | t.Fatal(err) |
| 74 | } |
| 75 | } |
| 76 | f := newRecoveryFixture(t) |
| 77 | obs := f.read() |
| 78 | h := obs.Claim.Holder |
| 79 | plan14PutRecord(t, f.claims, provision.Record{Schema: 1, Host: obs.Host, Account: h.Account, App: h.App, Channel: "stable", CreatedAt: f.now}) |
| 80 | store := release.NewDirStore(filepath.Join(dir, "deployment")) |
| 81 | keys, err := f.claims.List(t.Context(), "") |
| 82 | if err != nil { |
| 83 | t.Fatal(err) |
| 84 | } |
| 85 | for _, key := range keys { |
| 86 | body, _, err := f.claims.Get(t.Context(), key) |
| 87 | if err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | if _, err = store.Put(t.Context(), key, body, ""); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | } |
| 94 | cmd := exec.Command(os.Args[0], "-test.run=^TestPlan14InventoryProcessCrash$") |
| 95 | cmd.Env = append(os.Environ(), "PLAN14_INVENTORY_CRASH="+phase, "PLAN14_INVENTORY_DIR="+dir, "TMPDIR="+filepath.Join(dir, "child")) |
| 96 | output, err := cmd.CombinedOutput() |
| 97 | var exit *exec.ExitError |
| 98 | if !errors.As(err, &exit) || exit.ExitCode() != 73 { |
| 99 | t.Fatalf("child: %v %s", err, output) |
| 100 | } |
| 101 | root, _ := plan14InventoryProcessRoot(t, dir) |
| 102 | root.Journals.Now = func() time.Time { return time.Now().Add(25 * time.Hour) } |
| 103 | if err = root.recoverRootInventory(t.Context()); err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | captured := hostguard.Captured{Incarnation: h.Incarnation, Stamp: h.Stamp} |
| 107 | view, err := root.Stop.Coordinator.Guard.Root.View(t.Context(), obs.Host, captured) |
| 108 | if err != nil || view.Installation.InventoryRevision != 1 { |
| 109 | t.Fatal(view, err) |
| 110 | } |
| 111 | _, revision, err := root.Journals.Inventory.Snapshot(t.Context(), obs.Host, captured) |
| 112 | if err != nil || revision != 1 { |
| 113 | t.Fatal(revision, err) |
| 114 | } |
| 115 | if err = root.recoverRootInventory(t.Context()); err != nil { |
| 116 | t.Fatal("replay", err) |
| 117 | } |
| 118 | }) |
| 119 | } |
| 120 | } |
| 121 | |
amadan