| 1 | //go:build linux |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "database/sql" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "github.com/carlosframework/platform/internal/hostclaim" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "reflect" |
| 14 | "testing" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/hostbox" |
| 17 | "github.com/carlosframework/platform/internal/hostguard" |
| 18 | ) |
| 19 | |
| 20 | func TestServingHandoverRebootDirectoryProof(t *testing.T) { |
| 21 | for _, fault := range []string{"", "same-boot", "same-boot-renumber", "missing-boot", "boot-read", "foreign-host", "inode", "handle", "mount", "missing-record"} { |
| 22 | t.Run(fault, func(t *testing.T) { |
| 23 | f, r := servingRebootPreparationFixture(t, "complete") |
| 24 | if err := f.p.commitServingReboot(t.Context(), r.Authorization.ID); err != nil { |
| 25 | t.Fatal(err) |
| 26 | } |
| 27 | fs := f.p.ledger.fs.(*servingParkFS) |
| 28 | prior := hostguard.FenceTarget{Host: r.Authorization.Successor.Host, Box: r.Authorization.Successor.Box, Incarnation: r.Authorization.Successor.Incarnation, Stamp: r.Authorization.Successor.Stamp, BootID: r.Source.BootID} |
| 29 | e := hostbox.StopInventoryEntry{Kind: "path", ManagedRoot: "data", RelativePath: "instances/" + prior.Host, Device: r.Source.Directory.Device, Inode: r.Source.Directory.Inode, Type: 0040000} |
| 30 | a := hostbox.Artifact{Dev: r.Source.Directory.Device, Ino: e.Inode, Type: e.Type} |
| 31 | switch fault { |
| 32 | case "same-boot", "same-boot-renumber": |
| 33 | prior.BootID = servingOtherBoot |
| 34 | if fault == "same-boot-renumber" { |
| 35 | e.Device++ |
| 36 | } |
| 37 | case "missing-boot": |
| 38 | prior.BootID = "" |
| 39 | case "boot-read": |
| 40 | fs.boot = func() (string, error) { return "", os.ErrPermission } |
| 41 | case "foreign-host": |
| 42 | prior.Host = "foreign.example" |
| 43 | case "inode": |
| 44 | e.Inode++ |
| 45 | case "handle": |
| 46 | original := fs.handle |
| 47 | fs.handle = func(d *os.File) (servingParkPersistentDirectory, error) { |
| 48 | p, err := original(d) |
| 49 | if filepath.Base(d.Name()) == prior.Host { |
| 50 | p.Handle += "00" |
| 51 | } |
| 52 | return p, err |
| 53 | } |
| 54 | case "mount": |
| 55 | binding, err := servingParkReadBinding(f.disk, "namespace") |
| 56 | if err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | binding.Parent.Mount++ |
| 60 | if err := f.p.ledger.write("current-namespace.json", binding); err != nil { |
| 61 | t.Fatal(err) |
| 62 | } |
| 63 | case "missing-record": |
| 64 | if err := f.disk.Remove("retirement-" + r.Authorization.ID + ".json"); err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | } |
| 68 | h := servingTransportRoot(t, f) |
| 69 | err := h.verifyServingHandoverDirectory(t.Context(), prior, e, a) |
| 70 | if (err == nil) != (fault == "" || fault == "same-boot") { |
| 71 | t.Fatalf("fault %q: %v", fault, err) |
| 72 | } |
| 73 | }) |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | func TestServingHandoverRebootEqualNumbers(t *testing.T) { |
| 78 | for _, changed := range []bool{false, true} { |
| 79 | t.Run(fmt.Sprint(changed), func(t *testing.T) { |
| 80 | f := newServingRootFixture(t, true) |
| 81 | root := servingHandoverRoot(t, f) |
| 82 | next := func() hostbox.ColdSuccessor { |
| 83 | return hostbox.ColdSuccessor{Host: f.current.Host, Incarnation: f.current.Incarnation, Box: f.actor.Box, Enrollment: f.actor.Enrollment, DirectoryRevision: "7", ETag: f.f.read().ETag, Generation: f.current.Generation, Stamp: f.current.Stamp, Session: f.current.Session} |
| 84 | } |
| 85 | if err := root.servingHandover().Run(t.Context(), f.receipt.Target, next()); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | a, err := f.park() |
| 89 | if err != nil { |
| 90 | t.Fatal(err) |
| 91 | } |
| 92 | f.tags(f.current.Stamp) |
| 93 | if err := f.p.completeServingDatabasePark(t.Context(), f.conn, a.ID, f.current); err != nil { |
| 94 | t.Fatal(err) |
| 95 | } |
| 96 | prior := servingStop(t, f.f, f.f.read(), f.receipt.Target.BootID).Target |
| 97 | o := f.f.read() |
| 98 | admitted, err := hostclaim.Update(t.Context(), f.f.claims, o, hostclaim.AdmitWriter(hostclaim.Admission{ObservedSession: f.current.Session, ObservedGeneration: f.current.Generation, Box: f.actor.Box, Region: f.actor.Region, PredecessorRegion: f.actor.Region, Mode: "cold", NewGeneration: "G3"})) |
| 99 | if err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | f.current, err = servingWriterIdentity(admitted, f.actor.Box, f.actor.Region) |
| 103 | if err != nil { |
| 104 | t.Fatal(err) |
| 105 | } |
| 106 | fs := f.p.ledger.fs.(*servingParkFS) |
| 107 | fs.boot = func() (string, error) { return servingOtherBoot, nil } |
| 108 | if prior.BootID == servingOtherBoot { |
| 109 | t.Fatal("boots must differ") |
| 110 | } |
| 111 | if err := f.p.commitServingReboot(t.Context(), a.ID); err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | if changed { |
| 115 | handle := fs.handle |
| 116 | fs.handle = func(d *os.File) (servingParkPersistentDirectory, error) { |
| 117 | p, err := handle(d) |
| 118 | if filepath.Base(d.Name()) == prior.Host { |
| 119 | p.Handle += "00" |
| 120 | } |
| 121 | return p, err |
| 122 | } |
| 123 | } |
| 124 | h := root.servingHandover() |
| 125 | verify := h.VerifyDirectory |
| 126 | calls := 0 |
| 127 | h.VerifyDirectory = func(ctx context.Context, p hostguard.FenceTarget, e hostbox.StopInventoryEntry, a hostbox.Artifact) error { |
| 128 | calls++ |
| 129 | if a.Dev != e.Device || a.Ino != e.Inode || a.Type != e.Type { |
| 130 | t.Fatal("numeric identities must be equal") |
| 131 | } |
| 132 | return verify(ctx, p, e, a) |
| 133 | } |
| 134 | before, err := h.Registry.Get(prior.Host) |
| 135 | if err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | n := next() |
| 139 | err = h.Run(t.Context(), prior, n) |
| 140 | if changed { |
| 141 | if !errors.Is(err, hostbox.ErrChanged) { |
| 142 | t.Fatal("changed persistent evidence accepted", err) |
| 143 | } |
| 144 | after, err := h.Registry.Get(prior.Host) |
| 145 | if err != nil || !reflect.DeepEqual(before, after) { |
| 146 | t.Fatal("registry changed", err) |
| 147 | } |
| 148 | if _, err := h.Inventory.ReadInstallation(t.Context(), n.Host, n.Incarnation, n.Stamp); !errors.Is(err, sql.ErrNoRows) { |
| 149 | t.Fatal("successor inventory written", err) |
| 150 | } |
| 151 | } else { |
| 152 | if err != nil { |
| 153 | t.Fatal("unchanged filesystem refused", err) |
| 154 | } |
| 155 | if err := h.Run(t.Context(), prior, n); err != nil { |
| 156 | t.Fatal("replay", err) |
| 157 | } |
| 158 | if calls < 5 { |
| 159 | t.Fatal("missing boundary verification", calls) |
| 160 | } |
| 161 | } |
| 162 | if calls == 0 { |
| 163 | t.Fatal("persistent verifier bypassed") |
| 164 | } |
| 165 | }) |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | func servingRecoveryFilesystem(t *testing.T, data string) (string, *servingParkRuntime) { |
| 170 | t.Helper() |
| 171 | disk := &rollbackDisk{dir: t.TempDir()} |
| 172 | fs := newServingParkFS(data, disk) |
| 173 | boot, err := fs.currentBoot() |
| 174 | if err != nil { |
| 175 | t.Fatal(err) |
| 176 | } |
| 177 | return boot, &servingParkRuntime{authority: &servingParkRoot{ledger: &servingParkLedger{state: disk, fs: fs}}} |
| 178 | } |
| 179 | |
amadan