| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "sync/atomic" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "crypto/ed25519" |
| 14 | "github.com/carlosframework/platform/internal/hostbox" |
| 15 | "github.com/carlosframework/platform/internal/hostclaim" |
| 16 | "github.com/carlosframework/platform/internal/hostguard" |
| 17 | "github.com/carlosframework/platform/internal/hostwrite" |
| 18 | "github.com/carlosframework/platform/internal/registry" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | "github.com/carlosframework/platform/internal/rootstate" |
| 21 | ) |
| 22 | |
| 23 | type plan14HeldStore struct { |
| 24 | *rollbackDisk |
| 25 | token chan struct{} |
| 26 | attempts atomic.Int32 |
| 27 | } |
| 28 | |
| 29 | func (s *plan14HeldStore) Lock(ctx context.Context, _ string) (func(), error) { |
| 30 | s.attempts.Add(1) |
| 31 | select { |
| 32 | case s.token <- struct{}{}: |
| 33 | return func() { <-s.token }, nil |
| 34 | case <-ctx.Done(): |
| 35 | return nil, ctx.Err() |
| 36 | } |
| 37 | } |
| 38 | func (s *plan14HeldStore) Close() error { return nil } |
| 39 | |
| 40 | type plan14HeldReader struct{ f *hrFixture } |
| 41 | |
| 42 | func (r plan14HeldReader) Read(ctx context.Context, host string) (hostclaim.Observed, error) { |
| 43 | o, err := r.f.claims(ctx, host) |
| 44 | o.ETag = "claim-etag" |
| 45 | return o, err |
| 46 | } |
| 47 | |
| 48 | func plan14HeldRoot(t *testing.T) (*hrFixture, hostguard.Guard, *plan14HeldStore, *plan14HeldStore) { |
| 49 | t.Helper() |
| 50 | f := newHRFixture(t) |
| 51 | hosts := &plan14HeldStore{rollbackDisk: &rollbackDisk{dir: t.TempDir()}, token: make(chan struct{}, 1)} |
| 52 | releases := &plan14HeldStore{rollbackDisk: &rollbackDisk{dir: t.TempDir()}, token: make(chan struct{}, 1)} |
| 53 | g := hostguard.Guard{DataDir: f.data, LockDir: t.TempDir(), Claims: plan14HeldReader{f}, Root: hostguard.NewRootAuthority(hosts)} |
| 54 | h := f.claim.Holder |
| 55 | in := hostguard.RootInstallation{Host: f.claim.Host, Incarnation: h.Incarnation, Stamp: h.Stamp, Account: h.Account, App: h.App, Record: h.Key, Box: "uk-box", Enrollment: "en", Migration: 1} |
| 56 | if err := g.EstablishRoot(t.Context(), in, func(*hostclaim.Claim) error { return nil }); err != nil { |
| 57 | t.Fatal(err) |
| 58 | } |
| 59 | if err := os.MkdirAll(filepath.Join(g.DataDir, "hostguard"), 0700); err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | agent := g |
| 63 | agent.Root = nil |
| 64 | if err := agent.Activate(t.Context(), f.claim.Host, "uk-box", nil); err != nil { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | f.root.OpenRelease = func() (rootstate.Store, error) { return releases, nil } |
| 68 | source := hostbox.WorkSource{Guard: g, Box: "uk-box", Region: "uk", Now: time.Now, Local: func(context.Context, *hostclaim.Claim, hostbox.Selection) error { return nil }, Policy: MovesClaimPolicy(f.records)} |
| 69 | f.root.Works = func(ctx context.Context, _ hostbox.Actor, host, action string) (hostwrite.Work, error) { |
| 70 | return source.Select(ctx, host, action) |
| 71 | } |
| 72 | f.root.Selected = hostbox.Selected |
| 73 | return f, g, hosts, releases |
| 74 | } |
| 75 | |
| 76 | func TestPlan14HeldReplayChecksRootFence(t *testing.T) { |
| 77 | for _, attack := range []string{"delete", "rename", "replace lock", "forge"} { |
| 78 | t.Run(attack, func(t *testing.T) { |
| 79 | f, g, _, _ := plan14HeldRoot(t) |
| 80 | q := f.request(heldReleasePublish) |
| 81 | if _, err := f.root.Serve(t.Context(), hrPeer, q); err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | if out, err := f.root.Serve(t.Context(), hrPeer, q); err != nil || !out.Replayed { |
| 85 | t.Fatal(out, err) |
| 86 | } |
| 87 | target := hostguard.FenceTarget{Host: q.Host, Incarnation: q.Incarnation, Stamp: q.Stamp, Session: q.Session, Box: q.Box, Enrollment: "en", FenceID: "later-root-fence", BootID: "boot", ControllerID: "edge", ClaimETag: "claim-etag", DirectoryRevision: "1", Op: q.MoveID, Phase: q.Phase} |
| 88 | if _, err := g.BeginFence(t.Context(), target); err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | path := filepath.Join(g.DataDir, "hostguard", q.Host+".json") |
| 92 | var err error |
| 93 | switch attack { |
| 94 | case "delete": |
| 95 | err = os.Remove(path) |
| 96 | case "rename": |
| 97 | err = os.Rename(filepath.Dir(path), filepath.Dir(path)+".old") |
| 98 | case "replace lock": |
| 99 | err = os.Rename(g.LockDir, g.LockDir+".old") |
| 100 | if err == nil { |
| 101 | err = os.Mkdir(g.LockDir, 0700) |
| 102 | } |
| 103 | case "forge": |
| 104 | err = os.WriteFile(path, []byte(`{}`), 0600) |
| 105 | } |
| 106 | if err != nil { |
| 107 | t.Fatal(err) |
| 108 | } |
| 109 | if _, err := f.root.Serve(t.Context(), hrPeer, q); !errors.Is(err, hostguard.ErrFenced) { |
| 110 | t.Fatalf("fenced replay: %v", err) |
| 111 | } |
| 112 | pending, err := g.PendingFences(t.Context()) |
| 113 | if err != nil || len(pending) != 1 || pending[0] != target { |
| 114 | t.Fatal(pending, err) |
| 115 | } |
| 116 | }) |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | func TestPlan14AdoptPublishNoLockInversion(t *testing.T) { |
| 121 | for _, first := range []string{"adopt", "publish", "cancel"} { |
| 122 | t.Run(first, func(t *testing.T) { |
| 123 | f, g, hosts, releases := plan14HeldRoot(t) |
| 124 | ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) |
| 125 | defer cancel() |
| 126 | entered := make(chan struct{}) |
| 127 | unblock := make(chan struct{}) |
| 128 | var active, effects atomic.Int32 |
| 129 | effect := func(wait bool) error { |
| 130 | if active.Add(1) != 1 { |
| 131 | active.Add(-1) |
| 132 | return errors.New("overlapping root effects") |
| 133 | } |
| 134 | defer active.Add(-1) |
| 135 | if wait { |
| 136 | close(entered) |
| 137 | select { |
| 138 | case <-unblock: |
| 139 | case <-ctx.Done(): |
| 140 | return ctx.Err() |
| 141 | } |
| 142 | } |
| 143 | effects.Add(1) |
| 144 | return nil |
| 145 | } |
| 146 | f.root.materialize = func(ctx context.Context, _ release.Store, _ ed25519.PublicKey, _ *release.DigestRatchet, _ string, _ registry.Route, _ string, _ *release.ProofGate, w hostwrite.Work, lock func(func() error) error) (*release.Manifest, error) { |
| 147 | err := w.With(ctx, hostwrite.Identity{Release: "v7"}, func() error { return lock(func() error { return effect(first == "publish") }) }) |
| 148 | return nil, err |
| 149 | } |
| 150 | adopt := func() error { |
| 151 | return withRootReleaseStore(ctx, f.root.OpenRelease, func() error { |
| 152 | captured, err := g.Capture(f.claim.Host) |
| 153 | if err != nil { |
| 154 | return err |
| 155 | } |
| 156 | return g.With(ctx, f.claim.Host, captured, func(*hostclaim.Claim) error { return nil }, func() error { return effect(first != "publish") }) |
| 157 | }) |
| 158 | } |
| 159 | publish := func(c context.Context) error { |
| 160 | _, err := f.root.Serve(c, hrPeer, f.request(heldReleasePublish)) |
| 161 | return err |
| 162 | } |
| 163 | firstDone := make(chan error, 1) |
| 164 | if first == "publish" { |
| 165 | go func() { firstDone <- publish(ctx) }() |
| 166 | } else { |
| 167 | go func() { firstDone <- adopt() }() |
| 168 | } |
| 169 | select { |
| 170 | case <-entered: |
| 171 | case err := <-firstDone: |
| 172 | t.Fatal("first failed", err) |
| 173 | case <-ctx.Done(): |
| 174 | t.Fatal(ctx.Err()) |
| 175 | } |
| 176 | hostAttempts := hosts.attempts.Load() |
| 177 | releaseAttempts := releases.attempts.Load() |
| 178 | secondDone := make(chan error, 1) |
| 179 | waiting, stop := context.WithCancel(ctx) |
| 180 | defer stop() |
| 181 | if first == "publish" { |
| 182 | go func() { secondDone <- adopt() }() |
| 183 | } else { |
| 184 | go func() { secondDone <- publish(waiting) }() |
| 185 | } |
| 186 | for releases.attempts.Load() == releaseAttempts { |
| 187 | select { |
| 188 | case err := <-secondDone: |
| 189 | t.Fatal("waiter escaped release lock", err) |
| 190 | case <-ctx.Done(): |
| 191 | t.Fatal("release was not requested", ctx.Err()) |
| 192 | case <-time.After(time.Millisecond): |
| 193 | } |
| 194 | } |
| 195 | if hosts.attempts.Load() != hostAttempts { |
| 196 | t.Fatal("waiter entered host lock before release") |
| 197 | } |
| 198 | if first == "cancel" { |
| 199 | stop() |
| 200 | select { |
| 201 | case err := <-secondDone: |
| 202 | if !errors.Is(err, context.Canceled) { |
| 203 | t.Fatal(err) |
| 204 | } |
| 205 | case <-ctx.Done(): |
| 206 | t.Fatal("canceled publisher stuck") |
| 207 | } |
| 208 | if effects.Load() != 0 { |
| 209 | t.Fatal("canceled publisher performed effects") |
| 210 | } |
| 211 | if _, err := os.Stat(f.root.receiptPath(f.claim.Host)); !errors.Is(err, os.ErrNotExist) { |
| 212 | t.Fatal("canceled publisher wrote receipt", err) |
| 213 | } |
| 214 | } |
| 215 | close(unblock) |
| 216 | if err := <-firstDone; err != nil { |
| 217 | t.Fatal(err) |
| 218 | } |
| 219 | if first != "cancel" { |
| 220 | if err := <-secondDone; err != nil { |
| 221 | t.Fatal(err) |
| 222 | } |
| 223 | if effects.Load() != 2 { |
| 224 | t.Fatal(effects.Load()) |
| 225 | } |
| 226 | } |
| 227 | }) |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | func TestPlan14HeldRevalidatesCanonicalBindingUnderGuard(t *testing.T) { |
| 232 | for _, replay := range []bool{false, true} { |
| 233 | for _, changed := range []string{"record incarnation", "record key"} { |
| 234 | t.Run(fmt.Sprintf("%t/%s", replay, changed), func(t *testing.T) { |
| 235 | f, _, _, _ := plan14HeldRoot(t) |
| 236 | q := f.request(heldReleasePublish) |
| 237 | if replay { |
| 238 | if _, err := f.root.Serve(t.Context(), hrPeer, q); err != nil { |
| 239 | t.Fatal(err) |
| 240 | } |
| 241 | } |
| 242 | original := f.root.Works |
| 243 | f.root.Works = func(ctx context.Context, actor hostbox.Actor, host, action string) (hostwrite.Work, error) { |
| 244 | w, err := original(ctx, actor, host, action) |
| 245 | f.set(func(f *hrFixture) { |
| 246 | if changed == "record incarnation" { |
| 247 | f.rec.CreatedAt = f.rec.CreatedAt.Add(time.Second) |
| 248 | } else { |
| 249 | f.claim.Holder.Key = "apps/bab/other/instances/" + q.Host + ".json" |
| 250 | } |
| 251 | }) |
| 252 | return w, err |
| 253 | } |
| 254 | effects := 0 |
| 255 | f.root.materialize = func(ctx context.Context, _ release.Store, _ ed25519.PublicKey, _ *release.DigestRatchet, _ string, _ registry.Route, _ string, _ *release.ProofGate, w hostwrite.Work, _ func(func() error) error) (*release.Manifest, error) { |
| 256 | return nil, w.With(ctx, hostwrite.Identity{Release: q.Version}, func() error { effects++; return nil }) |
| 257 | } |
| 258 | if out, err := f.root.Serve(t.Context(), hrPeer, q); !errors.Is(err, hostbox.ErrChanged) || out.Replayed || effects != 0 { |
| 259 | t.Fatalf("changed binding admitted: %+v %v effects=%d", out, err, effects) |
| 260 | } |
| 261 | }) |
| 262 | } |
| 263 | } |
| 264 | } |
| 265 | |
amadan