| 1 | //go:build unix |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "bytes" |
| 7 | "context" |
| 8 | "crypto/sha256" |
| 9 | "encoding/hex" |
| 10 | "encoding/json" |
| 11 | "errors" |
| 12 | "fmt" |
| 13 | "os" |
| 14 | "path/filepath" |
| 15 | "slices" |
| 16 | "strings" |
| 17 | "testing" |
| 18 | "time" |
| 19 | |
| 20 | "github.com/carlosframework/platform/internal/activator" |
| 21 | "github.com/carlosframework/platform/internal/hostbox" |
| 22 | "github.com/carlosframework/platform/internal/hostclaim" |
| 23 | "github.com/carlosframework/platform/internal/hostguard" |
| 24 | "github.com/carlosframework/platform/internal/hostrelocate" |
| 25 | ) |
| 26 | |
| 27 | // stopProbeActivator is a real activator.Service's fences, the calls |
| 28 | // recorded in order. Replacing svc is an agent restart: the activator's |
| 29 | // fences live only in its process. failNext, when set, is the next |
| 30 | // FenceAndStopAs's answer instead of the service's. |
| 31 | type stopProbeActivator struct { |
| 32 | order *[]string |
| 33 | svc *activator.Service |
| 34 | failNext error |
| 35 | } |
| 36 | |
| 37 | func newStopProbeActivator(order *[]string) *stopProbeActivator { |
| 38 | return &stopProbeActivator{order: order, svc: &activator.Service{}} |
| 39 | } |
| 40 | |
| 41 | func (a *stopProbeActivator) ProbeWake(context.Context, activator.ProbeConfig) (activator.ProbeResult, error) { |
| 42 | return activator.ProbeResult{}, errors.New("not a probe") |
| 43 | } |
| 44 | |
| 45 | func (a *stopProbeActivator) FenceAndStopAs(ctx context.Context, host string, generation uint64, owner string) (*activator.StopEvidence, error) { |
| 46 | *a.order = append(*a.order, "fence-as") |
| 47 | if err := a.failNext; err != nil { |
| 48 | a.failNext = nil |
| 49 | return nil, err |
| 50 | } |
| 51 | return a.svc.FenceAndStopAs(ctx, host, generation, owner) |
| 52 | } |
| 53 | |
| 54 | func (a *stopProbeActivator) LiftFenceAs(host string, generation uint64, owner string) bool { |
| 55 | *a.order = append(*a.order, "lift") |
| 56 | return a.svc.LiftFenceAs(host, generation, owner) |
| 57 | } |
| 58 | |
| 59 | // foreignFence installs another operation's plain fence on a.svc. |
| 60 | func (a *stopProbeActivator) foreignFence(t *testing.T, host string, generation uint64) { |
| 61 | t.Helper() |
| 62 | ctx, cancel := context.WithTimeout(t.Context(), time.Second) |
| 63 | defer cancel() |
| 64 | if _, err := a.svc.FenceAndStop(ctx, host, generation); err != nil && !errors.Is(err, activator.ErrStopUnproven) { |
| 65 | t.Fatal(err) |
| 66 | } |
| 67 | } |
| 68 | |
| 69 | // stopProbeFixture is a target at relocating/probing whose probe passed |
| 70 | // with session 12: the guard activated at the install stamp, main's |
| 71 | // Wake.Stop standing for root's writer-session stop (it fences and |
| 72 | // completes the stop in the guard journal with a receipt), and the probe |
| 73 | // stop's provider bound to them. |
| 74 | type stopProbeFixture struct { |
| 75 | t *testing.T |
| 76 | order []string |
| 77 | act *stopProbeActivator |
| 78 | a ClaimAssembly |
| 79 | cfg RelocationAssemblyConfig |
| 80 | obs hostclaim.Observed |
| 81 | receipt []byte |
| 82 | target hostguard.FenceTarget |
| 83 | } |
| 84 | |
| 85 | func newStopProbeFixture(t *testing.T) *stopProbeFixture { |
| 86 | t.Helper() |
| 87 | ctx := t.Context() |
| 88 | const host, box = "shop.example", "dst" |
| 89 | created := time.Date(2026, 10, 7, 0, 0, 0, 0, time.UTC) |
| 90 | inc, err := hostclaim.Incarnation("acct", "app", host, created) |
| 91 | if err != nil { |
| 92 | t.Fatal(err) |
| 93 | } |
| 94 | probe, _ := json.Marshal(hostclaim.Probe{Box: box, Session: 12, At: created.Add(time.Hour)}) |
| 95 | f := &stopProbeFixture{t: t} |
| 96 | f.obs = hostclaim.Observed{Host: host, ETag: "e1", Claim: &hostclaim.Claim{V: 2, Gen: 30, Host: host, |
| 97 | Holder: &hostclaim.Holder{Family: "instance", Key: "apps/acct/app/instances/" + host + ".json", Account: "acct", App: "app", Incarnation: inc, CreatedAt: created, |
| 98 | Location: "instances/" + host, Stamp: 9, Session: 12, Generation: "P1", WriterBox: box, Admissions: []hostclaim.AdmissionRecord{{Session: 12, Box: box, Generation: "P1", Mode: "cold", Region: "ie"}}}, |
| 99 | Op: hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Exec: box, Participants: []string{"src", box}, Journal: map[string]string{"probe": string(probe)}}}} |
| 100 | dataDir := t.TempDir() |
| 101 | if err := os.Mkdir(filepath.Join(dataDir, "hostguard"), 0o770); err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | guard := hostguard.Guard{DataDir: dataDir, LockDir: t.TempDir(), Claims: &guardClaims{obs: f.obs}} |
| 105 | if err := guard.Activate(ctx, host, box, nil); err != nil { |
| 106 | t.Fatal(err) |
| 107 | } |
| 108 | f.target = hostguard.FenceTarget{FenceID: "fence-1", Region: "ie", Host: host, Incarnation: inc, Stamp: 9, Session: 12, Op: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, |
| 109 | ObligationID: hostguard.WriterStopObligation(12), Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", ClaimETag: "e1", DirectoryRevision: "1", RosterRevision: 1} |
| 110 | at := created.Add(2 * time.Hour) |
| 111 | f.receipt = mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r1", Kind: StopReceiptStopped, Region: "ie", ClaimETag: "e1", ObligationID: f.target.ObligationID, FenceID: "fence-1", Host: host, Incarnation: inc, |
| 112 | InstallationStamp: 9, WriterSession: 12, OperationID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Box: box, Enrollment: "enr", BootID: "boot", ControllerID: "ctl", DirectoryRevision: testReceiptNumeral(f.target), Binding: "1", EvidenceAt: at}) |
| 113 | wake := &ClaimWakeAssembly{Stop: func(ctx context.Context, o hostclaim.Observed) error { |
| 114 | f.order = append(f.order, "wake-stop") |
| 115 | // Root's writer-session stop: deterministic, so a repeat meets the |
| 116 | // same, already completed, fence. |
| 117 | if got, err := guard.CompletedWriterStops(ctx, host, hostguard.Captured{Incarnation: inc, Stamp: 9}, "rel-1", box); err == nil && len(got) == 1 { |
| 118 | return nil |
| 119 | } |
| 120 | if _, err := guard.BeginFence(ctx, f.target); err != nil { |
| 121 | return err |
| 122 | } |
| 123 | return guard.CompleteFence(ctx, f.target, f.receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }) |
| 124 | }} |
| 125 | f.act = newStopProbeActivator(&f.order) |
| 126 | f.a = ClaimAssembly{Guard: guard, Wake: wake, stopLocal: func(context.Context) (claimStopLocal, error) { |
| 127 | return claimStopLocal{Actor: hostbox.Actor{Box: box, Enrollment: "enr", Region: "ie", Revision: 1, Binding: "1"}, BootID: "boot", ControllerID: "ctl", PID: 100, ProcessStart: "5000"}, nil |
| 128 | }} |
| 129 | f.cfg = RelocationAssemblyConfig{Activator: f.act, DataDir: dataDir} |
| 130 | return f |
| 131 | } |
| 132 | |
| 133 | func (f *stopProbeFixture) stop() (hostclaim.ProbeStop, error) { |
| 134 | return f.a.relocationStopProbe(f.cfg)(f.t.Context(), f.obs) |
| 135 | } |
| 136 | |
| 137 | // marker is the probe stop's persisted owner for rel-1. |
| 138 | func (f *stopProbeFixture) marker() relocationProbeFence { |
| 139 | f.t.Helper() |
| 140 | path, err := relocationProbeFencePath(f.cfg.DataDir, "rel-1") |
| 141 | if err != nil { |
| 142 | f.t.Fatal(err) |
| 143 | } |
| 144 | rec, ok, err := readRelocationProbeFence(path) |
| 145 | if err != nil || !ok { |
| 146 | f.t.Fatalf("no probe-fence marker: %v", err) |
| 147 | } |
| 148 | return rec |
| 149 | } |
| 150 | |
| 151 | // Ruling 49 steps 1 to 3 with ruling 75's owned fence, against the real |
| 152 | // activator: the owner is persisted, then the fence goes in before root is |
| 153 | // asked; an unproven stop (the probe's entry is gone) still asks root; |
| 154 | // root's receipt is the guard journal's writer-session stop of the probe's |
| 155 | // session; a repeat meets its own fence and returns the same ProbeStop; |
| 156 | // another operation's fence and an uninstalled fence stop everything with |
| 157 | // nothing asked of root and the other fence as it was. |
| 158 | func TestRelocationStopProbeFencesStopsAndReadsRootsReceipt(t *testing.T) { |
| 159 | sum := func(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) } |
| 160 | const host = "shop.example" |
| 161 | t.Run("the probe's session", func(t *testing.T) { |
| 162 | f := newStopProbeFixture(t) |
| 163 | s, err := f.stop() |
| 164 | if err != nil { |
| 165 | t.Fatal(err) |
| 166 | } |
| 167 | if want := (hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: sum(f.receipt)}); s != want { |
| 168 | t.Fatalf("stop %+v, want %+v", s, want) |
| 169 | } |
| 170 | if want := []string{"fence-as", "wake-stop"}; !slices.Equal(f.order, want) { |
| 171 | t.Fatalf("order %v, want %v", f.order, want) |
| 172 | } |
| 173 | m := f.marker() |
| 174 | if m.Host != host || m.Session != 12 || len(m.Owner) != 32 || f.act.svc.FenceGeneration(host) != 12 { |
| 175 | t.Fatalf("marker %+v, fence %d", m, f.act.svc.FenceGeneration(host)) |
| 176 | } |
| 177 | // A repeat (the claim CAS lost, or a later pass) meets its own |
| 178 | // fence under the same owner. |
| 179 | f.order = nil |
| 180 | again, err := f.stop() |
| 181 | if err != nil || again != s { |
| 182 | t.Fatalf("repeat: %+v %v", again, err) |
| 183 | } |
| 184 | if want := []string{"fence-as", "wake-stop"}; !slices.Equal(f.order, want) || f.marker() != m { |
| 185 | t.Fatalf("repeat order %v, marker %+v", f.order, f.marker()) |
| 186 | } |
| 187 | }) |
| 188 | for _, gen := range []uint64{5, 12, 20} { |
| 189 | t.Run(fmt.Sprintf("another operation's fence at %d", gen), func(t *testing.T) { |
| 190 | f := newStopProbeFixture(t) |
| 191 | f.act.foreignFence(t, host, gen) |
| 192 | if _, err := f.stop(); !errors.Is(err, activator.ErrFenceHeld) { |
| 193 | t.Fatalf("got %v, want ErrFenceHeld", err) |
| 194 | } |
| 195 | if want := []string{"fence-as"}; !slices.Equal(f.order, want) || f.act.svc.FenceGeneration(host) != gen { |
| 196 | t.Fatalf("root was asked or the fence changed: %v, fence %d", f.order, f.act.svc.FenceGeneration(host)) |
| 197 | } |
| 198 | }) |
| 199 | } |
| 200 | t.Run("no fence installed", func(t *testing.T) { |
| 201 | f := newStopProbeFixture(t) |
| 202 | f.act.failNext = activator.ErrFenceNotInstalled |
| 203 | if _, err := f.stop(); !errors.Is(err, activator.ErrFenceNotInstalled) { |
| 204 | t.Fatalf("got %v", err) |
| 205 | } |
| 206 | if want := []string{"fence-as"}; !slices.Equal(f.order, want) { |
| 207 | t.Fatalf("root was asked: %v", f.order) |
| 208 | } |
| 209 | }) |
| 210 | t.Run("not the probe's session", func(t *testing.T) { |
| 211 | f := newStopProbeFixture(t) |
| 212 | f.obs.Claim.Holder.Session = 13 |
| 213 | if _, err := f.stop(); !errors.Is(err, hostbox.ErrChanged) || len(f.order) != 0 { |
| 214 | t.Fatalf("%v %v", err, f.order) |
| 215 | } |
| 216 | }) |
| 217 | } |
| 218 | |
| 219 | // Astra round 6, finding 1: the agent restarts after the probe stop |
| 220 | // installed its fence and persisted its owner. The restart empties the |
| 221 | // activator's fences; the marker survives. Another operation then fences |
| 222 | // the host at a lower or equal generation. The surviving marker must not |
| 223 | // let the probe stop take that fence over: ErrFenceHeld, the other fence |
| 224 | // exactly as it was, root not asked. Once the other operation lifts its |
| 225 | // fence, the next pass installs this probe's own fence and completes. |
| 226 | func TestRelocationStopProbeAfterARestartNeverTakesAForeignFence(t *testing.T) { |
| 227 | const host = "shop.example" |
| 228 | for _, gen := range []uint64{5, 12} { |
| 229 | t.Run(fmt.Sprint(gen), func(t *testing.T) { |
| 230 | f := newStopProbeFixture(t) |
| 231 | first, err := f.stop() |
| 232 | if err != nil { |
| 233 | t.Fatal(err) |
| 234 | } |
| 235 | before := f.marker() |
| 236 | f.act.svc = &activator.Service{} // the restart |
| 237 | f.act.foreignFence(t, host, gen) |
| 238 | f.order = nil |
| 239 | if _, err := f.stop(); !errors.Is(err, activator.ErrFenceHeld) { |
| 240 | t.Fatalf("got %v, want ErrFenceHeld", err) |
| 241 | } |
| 242 | if want := []string{"fence-as"}; !slices.Equal(f.order, want) || f.act.svc.FenceGeneration(host) != gen || f.marker() != before { |
| 243 | t.Fatalf("order %v, fence %d, marker %+v", f.order, f.act.svc.FenceGeneration(host), f.marker()) |
| 244 | } |
| 245 | if f.act.svc.LiftFenceAs(host, gen, before.Owner) { |
| 246 | t.Fatal("the surviving owner lifted the other operation's fence") |
| 247 | } |
| 248 | if !f.act.svc.LiftFence(host, gen) { |
| 249 | t.Fatal("the other operation's fence was not as it installed it") |
| 250 | } |
| 251 | f.order = nil |
| 252 | again, err := f.stop() |
| 253 | if err != nil || again != first || f.act.svc.FenceGeneration(host) != 12 { |
| 254 | t.Fatalf("after the other fence went: %+v %v, fence %d", again, err, f.act.svc.FenceGeneration(host)) |
| 255 | } |
| 256 | }) |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | // Astra round 6, finding 2: the owner is persisted before the activator is |
| 261 | // asked, so a failed write installs nothing and blocks nothing. The |
| 262 | // retry persists it and proceeds, under the owner it persisted. |
| 263 | func TestRelocationStopProbeRetriesAFailedMarkerWrite(t *testing.T) { |
| 264 | const host = "shop.example" |
| 265 | f := newStopProbeFixture(t) |
| 266 | path, err := relocationProbeFencePath(f.cfg.DataDir, "rel-1") |
| 267 | if err != nil { |
| 268 | t.Fatal(err) |
| 269 | } |
| 270 | // The op's journal directory cannot be made: a file stands in its way. |
| 271 | if err := os.MkdirAll(filepath.Dir(filepath.Dir(path)), 0o700); err != nil { |
| 272 | t.Fatal(err) |
| 273 | } |
| 274 | if err := os.WriteFile(filepath.Dir(path), []byte("in the way"), 0o600); err != nil { |
| 275 | t.Fatal(err) |
| 276 | } |
| 277 | if _, err := f.stop(); err == nil { |
| 278 | t.Fatal("the probe stop went on without its owner persisted") |
| 279 | } |
| 280 | if len(f.order) != 0 || f.act.svc.Fenced(host) { |
| 281 | t.Fatalf("the activator was asked before the owner was persisted: %v", f.order) |
| 282 | } |
| 283 | if err := os.Remove(filepath.Dir(path)); err != nil { |
| 284 | t.Fatal(err) |
| 285 | } |
| 286 | s, err := f.stop() |
| 287 | if err != nil || s.Session != 12 { |
| 288 | t.Fatalf("retry: %+v %v", s, err) |
| 289 | } |
| 290 | if want := []string{"fence-as", "wake-stop"}; !slices.Equal(f.order, want) { |
| 291 | t.Fatalf("retry order %v", f.order) |
| 292 | } |
| 293 | if m := f.marker(); !f.act.svc.LiftFenceAs(host, 12, m.Owner) { |
| 294 | t.Fatal("the fence installed is not the persisted owner's") |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | // Absence is never evidence: a probe-fence marker that is present is the |
| 299 | // only record of the owner a fence may already be installed under, so one |
| 300 | // that cannot be read, or that names another host, session or no owner, |
| 301 | // is never replaced by a fresh owner. The stop refuses before the |
| 302 | // activator is asked, no fence is installed, and the marker stays as it |
| 303 | // was, for an operator. |
| 304 | func TestRelocationStopProbeNeverReplacesAPresentMarker(t *testing.T) { |
| 305 | const host = "shop.example" |
| 306 | owner := strings.Repeat("a", 32) |
| 307 | for name, body := range map[string][]byte{ |
| 308 | "unreadable": []byte("{not json"), |
| 309 | "another session": []byte(`{"host":"shop.example","session":13,"owner":"` + owner + `"}`), |
| 310 | "another host": []byte(`{"host":"other.example","session":12,"owner":"` + owner + `"}`), |
| 311 | "no owner": []byte(`{"host":"shop.example","session":12}`), |
| 312 | } { |
| 313 | t.Run(name, func(t *testing.T) { |
| 314 | f := newStopProbeFixture(t) |
| 315 | path, err := relocationProbeFencePath(f.cfg.DataDir, "rel-1") |
| 316 | if err != nil { |
| 317 | t.Fatal(err) |
| 318 | } |
| 319 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| 320 | t.Fatal(err) |
| 321 | } |
| 322 | if err := os.WriteFile(path, body, 0o600); err != nil { |
| 323 | t.Fatal(err) |
| 324 | } |
| 325 | if _, err := f.stop(); !errors.Is(err, hostrelocate.ErrRefused) { |
| 326 | t.Fatalf("the probe stop over a marker it did not write: %v", err) |
| 327 | } |
| 328 | if len(f.order) != 0 || f.act.svc.Fenced(host) { |
| 329 | t.Fatalf("the activator was asked: %v", f.order) |
| 330 | } |
| 331 | if after, err := os.ReadFile(path); err != nil || !bytes.Equal(after, body) { |
| 332 | t.Fatalf("the marker was replaced: %q %v", after, err) |
| 333 | } |
| 334 | }) |
| 335 | } |
| 336 | } |
| 337 | |
| 338 | // Two completed writer-session stops of the probe's session under the same |
| 339 | // op on this box (a second fence under another roster revision) leave no |
| 340 | // one receipt to record or to hand over from. The stop and N6 refuse with |
| 341 | // hostguard.ErrFenceTarget, as main's CompletedColdPredecessor does, |
| 342 | // rather than take the first the journal lists. |
| 343 | func TestRelocationStopProbeRefusesTwoCompletedStopsOfTheSession(t *testing.T) { |
| 344 | ctx := t.Context() |
| 345 | f := newStopProbeFixture(t) |
| 346 | first, err := f.stop() |
| 347 | if err != nil { |
| 348 | t.Fatal(err) |
| 349 | } |
| 350 | // Root's second stop of session 12 under rel-1: a repeat under the next |
| 351 | // roster revision, which main's fence target carries, so the guard |
| 352 | // journals it beside the first rather than meeting it again. |
| 353 | second := f.target |
| 354 | second.FenceID, second.DirectoryRevision, second.RosterRevision = "fence-2", "2", 2 |
| 355 | if _, err := f.a.Guard.BeginFence(ctx, second); err != nil { |
| 356 | t.Fatal(err) |
| 357 | } |
| 358 | at := time.Date(2026, 10, 7, 4, 0, 0, 0, time.UTC) |
| 359 | receipt := mustMarshalStopReceipt(StopReceipt{V: 1, ID: "r2", Kind: StopReceiptStopped, Region: "ie", ClaimETag: "e1", ObligationID: second.ObligationID, FenceID: "fence-2", Host: f.obs.Host, Incarnation: second.Incarnation, |
| 360 | InstallationStamp: 9, WriterSession: 12, OperationID: "rel-1", Phase: hostclaim.PhaseRelocatingProbing, Box: "dst", Enrollment: "enr", BootID: "boot", ControllerID: "ctl", DirectoryRevision: testReceiptNumeral(second), Binding: "2", EvidenceAt: at}) |
| 361 | if err := f.a.Guard.CompleteFence(ctx, second, receipt, at, func(context.Context, hostguard.FenceTarget, []byte, time.Time) error { return nil }); err != nil { |
| 362 | t.Fatal(err) |
| 363 | } |
| 364 | if _, err := f.stop(); !errors.Is(err, hostguard.ErrFenceTarget) { |
| 365 | t.Fatalf("the stop over two receipts: %v", err) |
| 366 | } |
| 367 | // N6 over the serving record of the first stop refuses the same way. |
| 368 | done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: first, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| 369 | c := *f.obs.Claim |
| 370 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}} |
| 371 | h := relocationCompletedHandover{Guard: f.a.Guard, Box: "dst"} |
| 372 | if _, err := h.receipt(ctx, hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c}); !errors.Is(err, hostguard.ErrFenceTarget) { |
| 373 | t.Fatalf("N6 over two receipts: %v", err) |
| 374 | } |
| 375 | } |
| 376 | |
amadan