| 1 | //go:build unix |
| 2 | |
| 3 | package host |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "crypto/sha256" |
| 8 | "encoding/hex" |
| 9 | "encoding/json" |
| 10 | "errors" |
| 11 | "fmt" |
| 12 | "net" |
| 13 | "net/http" |
| 14 | "os" |
| 15 | "path/filepath" |
| 16 | "slices" |
| 17 | "testing" |
| 18 | |
| 19 | "github.com/carlosframework/platform/internal/hostbox" |
| 20 | "github.com/carlosframework/platform/internal/hostclaim" |
| 21 | execlease "github.com/carlosframework/platform/internal/hostexec" |
| 22 | "github.com/carlosframework/platform/internal/release" |
| 23 | ) |
| 24 | |
| 25 | // N6, ruling 56: wireClaimAdmission with the completed-relocation branch. |
| 26 | // The first serving wake after a relocation completed on this box hands |
| 27 | // over from the probe's existing receipt: before admission the receipt is |
| 28 | // found and verified and nothing is stopped; after it, root's |
| 29 | // /claim/handover is posted that receipt's target as the prior. Any other |
| 30 | // serving prior keeps main's stop and main's local handover. |
| 31 | func TestRelocationCompletedHandoverRoutesOnlyTheCompletedPrior(t *testing.T) { |
| 32 | ctx := t.Context() |
| 33 | f := newStopProbeFixture(t) |
| 34 | if _, err := f.stop(); err != nil { |
| 35 | t.Fatal(err) |
| 36 | } |
| 37 | sum := sha256.Sum256(f.receipt) |
| 38 | done, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: hex.EncodeToString(sum[:])}, |
| 39 | Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}) |
| 40 | serving := func(journal map[string]string) hostclaim.Observed { |
| 41 | c := *f.obs.Claim |
| 42 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, Journal: journal} |
| 43 | return hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c} |
| 44 | } |
| 45 | completedPrior := serving(map[string]string{hostclaim.JournalCompleted: string(done)}) |
| 46 | plainPrior := serving(nil) |
| 47 | admitted := func(prior hostclaim.Observed) hostclaim.Observed { |
| 48 | c := *prior.Claim |
| 49 | h := *c.Holder |
| 50 | h.Stamp, h.Session, h.Generation = 40, 40, "g40" |
| 51 | c.Holder = &h |
| 52 | return hostclaim.Observed{Host: prior.Host, ETag: "e3", Claim: &c} |
| 53 | } |
| 54 | |
| 55 | // Root's handover endpoint, recording what it is posted. |
| 56 | socket := filepath.Join(t.TempDir(), "c.sock") |
| 57 | l, err := net.Listen("unix", socket) |
| 58 | if err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | var posted []coldHandoverRequest |
| 62 | srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 63 | var req coldHandoverRequest |
| 64 | if r.URL.Path != "/claim/handover" || json.NewDecoder(r.Body).Decode(&req) != nil { |
| 65 | w.WriteHeader(http.StatusBadRequest) |
| 66 | return |
| 67 | } |
| 68 | posted = append(posted, req) |
| 69 | w.WriteHeader(http.StatusNoContent) |
| 70 | })} |
| 71 | go srv.Serve(l) |
| 72 | t.Cleanup(func() { srv.Close() }) |
| 73 | |
| 74 | stops, locals := 0, 0 |
| 75 | w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { stops++; return nil }} |
| 76 | local := func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil } |
| 77 | identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1, Binding: "1"}) |
| 78 | completed := &relocationCompletedHandover{Guard: f.a.Guard, Identity: identity, Socket: socket, Box: "dst"} |
| 79 | wireClaimAdmission(w, "dst", local, claimCrossBoxHandover{}, &ClaimRelocationOwners{}, completed) |
| 80 | |
| 81 | if err := w.BeforeAdmission(ctx, completedPrior); err != nil || stops != 0 { |
| 82 | t.Fatalf("before the completed prior's admission: %v (stops %d)", err, stops) |
| 83 | } |
| 84 | if err := w.AfterAdmission(ctx, completedPrior, admitted(completedPrior)); err != nil { |
| 85 | t.Fatalf("after the completed prior's admission: %v", err) |
| 86 | } |
| 87 | if len(posted) != 1 || posted[0].Prior != f.target || posted[0].Next.Session != 40 || posted[0].Next.Box != "dst" || locals != 0 { |
| 88 | t.Fatalf("root was posted %+v (locals %d)", posted, locals) |
| 89 | } |
| 90 | |
| 91 | if err := w.BeforeAdmission(ctx, plainPrior); err != nil || stops != 1 { |
| 92 | t.Fatalf("a serving prior without the record: %v (stops %d)", err, stops) |
| 93 | } |
| 94 | if err := w.AfterAdmission(ctx, plainPrior, admitted(plainPrior)); err != nil || locals != 1 || len(posted) != 1 { |
| 95 | t.Fatalf("a serving prior's handover: %v (locals %d, posted %d)", err, locals, len(posted)) |
| 96 | } |
| 97 | |
| 98 | // After the first serving wake the holder has moved on (session, stamp |
| 99 | // or generation), while the record still names the stopped probe: that |
| 100 | // prior is a serving session like any other, main's stop and main's |
| 101 | // local handover, never the probe's receipt again. |
| 102 | for name, move := range map[string]func(*hostclaim.Holder){ |
| 103 | "session": func(h *hostclaim.Holder) { h.Session = 40 }, |
| 104 | "stamp": func(h *hostclaim.Holder) { h.Stamp = 40 }, |
| 105 | "generation": func(h *hostclaim.Holder) { h.Generation = "g40" }, |
| 106 | } { |
| 107 | moved := completedPrior |
| 108 | c := *moved.Claim |
| 109 | h := *c.Holder |
| 110 | move(&h) |
| 111 | c.Holder = &h |
| 112 | moved.Claim = &c |
| 113 | if err := w.BeforeAdmission(ctx, moved); err != nil || stops != 2 { |
| 114 | t.Fatalf("%s moved on, before admission: %v (stops %d)", name, err, stops) |
| 115 | } |
| 116 | if err := w.AfterAdmission(ctx, moved, admitted(moved)); err != nil || locals != 2 || len(posted) != 1 { |
| 117 | t.Fatalf("%s moved on, after admission: %v (locals %d, posted %d)", name, err, locals, len(posted)) |
| 118 | } |
| 119 | stops, locals = 1, 1 |
| 120 | } |
| 121 | |
| 122 | // A record whose stop is not root's receipt is refused before anything. |
| 123 | forged := serving(map[string]string{hostclaim.JournalCompleted: string(mustJSON(t, hostclaim.CompletedRelocation{Op: "rel-1", |
| 124 | Stop: hostclaim.ProbeStop{Box: "dst", Session: 12, Receipt: "fence-1", ReceiptSHA256: hex.EncodeToString(make([]byte, 32))}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}))}) |
| 125 | if err := w.BeforeAdmission(ctx, forged); err == nil || stops != 1 { |
| 126 | t.Fatalf("a forged record: %v (stops %d)", err, stops) |
| 127 | } |
| 128 | |
| 129 | // A serving prior on this box whose record cannot be read is refused |
| 130 | // before and after admission, never routed to main's stop or local |
| 131 | // handover (absence is never evidence). |
| 132 | unreadable := serving(map[string]string{hostclaim.JournalCompleted: "{not json"}) |
| 133 | if err := w.BeforeAdmission(ctx, unreadable); !errors.Is(err, hostclaim.ErrRelocation) || stops != 1 { |
| 134 | t.Fatalf("an unreadable record before admission: %v (stops %d)", err, stops) |
| 135 | } |
| 136 | if err := w.AfterAdmission(ctx, unreadable, admitted(unreadable)); !errors.Is(err, hostclaim.ErrRelocation) || locals != 1 || len(posted) != 1 { |
| 137 | t.Fatalf("an unreadable record after admission: %v (locals %d, posted %d)", err, locals, len(posted)) |
| 138 | } |
| 139 | } |
| 140 | |
| 141 | // handoverRoot is root's /claim/handover endpoint on a unix socket, |
| 142 | // recording every request it is posted. |
| 143 | func handoverRoot(t *testing.T) (string, *[]coldHandoverRequest) { |
| 144 | t.Helper() |
| 145 | dir, err := os.MkdirTemp(os.TempDir(), "handover-") |
| 146 | if err != nil { |
| 147 | t.Fatal(err) |
| 148 | } |
| 149 | t.Cleanup(func() { os.RemoveAll(dir) }) |
| 150 | socket := filepath.Join(dir, "c.sock") |
| 151 | l, err := net.Listen("unix", socket) |
| 152 | if err != nil { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | posted := &[]coldHandoverRequest{} |
| 156 | srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 157 | var req coldHandoverRequest |
| 158 | if r.URL.Path != "/claim/handover" || json.NewDecoder(r.Body).Decode(&req) != nil { |
| 159 | w.WriteHeader(http.StatusBadRequest) |
| 160 | return |
| 161 | } |
| 162 | *posted = append(*posted, req) |
| 163 | w.WriteHeader(http.StatusNoContent) |
| 164 | })} |
| 165 | go srv.Serve(l) |
| 166 | t.Cleanup(func() { srv.Close() }) |
| 167 | return socket, posted |
| 168 | } |
| 169 | |
| 170 | // completedServing is the serving op CompleteRelocation leaves after the |
| 171 | // probe stop s of the fixture's relocating claim: the same holder, and |
| 172 | // the completed relocation journaled, marked discharged when the |
| 173 | // completion pass has run. |
| 174 | func (f *stopProbeFixture) completedServing(s hostclaim.ProbeStop, discharged bool) hostclaim.Observed { |
| 175 | f.t.Helper() |
| 176 | c := *f.obs.Claim |
| 177 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}, |
| 178 | Journal: map[string]string{hostclaim.JournalCompleted: string(mustJSON(f.t, hostclaim.CompletedRelocation{Op: "rel-1", Stop: s, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}))}} |
| 179 | if discharged { |
| 180 | if err := hostclaim.MarkCompletedDischarged("rel-1")(&c); err != nil { |
| 181 | f.t.Fatal(err) |
| 182 | } |
| 183 | } |
| 184 | return hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c} |
| 185 | } |
| 186 | |
| 187 | // Ruling 82's reasoning, applied to N6 (Task 9 review): a relocation |
| 188 | // completed while region HA was on, and region HA is now off. The claim |
| 189 | // assembly then has no relocation owners (and, with moves C1 off too, no |
| 190 | // cross-box routing at all), yet the first serving wake must still hand |
| 191 | // over from the probe's receipt: main's Wake.Stop would ask a |
| 192 | // serving-shaped re-stop of the stopped probe session. Through |
| 193 | // OpenProductionClaimAssembly's own wiring (wireProductionAdmission), the |
| 194 | // wake's predecessor stop finds and verifies the receipt with nothing |
| 195 | // stopped, and root is posted that receipt's target as the prior. A prior |
| 196 | // with no completion record, on a box that completed nothing, meets main's |
| 197 | // hooks exactly as they were. |
| 198 | func TestRelocationCompletedHandoverWithRegionHAOff(t *testing.T) { |
| 199 | ctx := t.Context() |
| 200 | for _, movesC1 := range []bool{true, false} { |
| 201 | for _, discharged := range []bool{false, true} { |
| 202 | t.Run(fmt.Sprintf("moves C1 %v, discharged %v", movesC1, discharged), func(t *testing.T) { |
| 203 | f := newStopProbeFixture(t) |
| 204 | s, err := f.stop() |
| 205 | if err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | f.order = nil |
| 209 | socket, posted := handoverRoot(t) |
| 210 | stops, locals := 0, 0 |
| 211 | w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { stops++; return nil }} |
| 212 | w.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil } |
| 213 | identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1, Binding: "1"}) |
| 214 | wireProductionAdmission(w, "dst", movesC1, claimCrossBoxHandover{Box: "dst"}, nil, |
| 215 | &relocationCompletedHandover{Guard: f.a.Guard, Identity: identity, Socket: socket, Box: "dst"}) |
| 216 | |
| 217 | prior := f.completedServing(s, discharged) |
| 218 | admitted := *prior.Claim |
| 219 | next := *admitted.Holder |
| 220 | next.Stamp, next.Session, next.Generation = 40, 40, "g40" |
| 221 | admitted.Holder = &next |
| 222 | if err := w.stopPredecessor(ctx, prior); err != nil || stops != 0 || len(f.order) != 0 { |
| 223 | t.Fatalf("the first serving wake's predecessor stop: %v (stops %d, root %v)", err, stops, f.order) |
| 224 | } |
| 225 | if err := w.AfterAdmission(ctx, prior, hostclaim.Observed{Host: prior.Host, ETag: "e3", Claim: &admitted}); err != nil { |
| 226 | t.Fatalf("the first serving wake's handover: %v", err) |
| 227 | } |
| 228 | if len(*posted) != 1 || (*posted)[0].Prior != f.target || (*posted)[0].Next.Session != 40 || locals != 0 { |
| 229 | t.Fatalf("root was posted %+v (locals %d)", *posted, locals) |
| 230 | } |
| 231 | }) |
| 232 | } |
| 233 | t.Run(fmt.Sprintf("moves C1 %v, no completion record", movesC1), func(t *testing.T) { |
| 234 | // A box that never completed a relocation: an empty guard |
| 235 | // journal, a serving prior with no record. |
| 236 | f := newStopProbeFixture(t) |
| 237 | socket, posted := handoverRoot(t) |
| 238 | stops, locals := 0, 0 |
| 239 | w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { stops++; return nil }} |
| 240 | w.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil } |
| 241 | identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1, Binding: "1"}) |
| 242 | wireProductionAdmission(w, "dst", movesC1, claimCrossBoxHandover{Box: "dst"}, nil, |
| 243 | &relocationCompletedHandover{Guard: f.a.Guard, Identity: identity, Socket: socket, Box: "dst"}) |
| 244 | c := *f.obs.Claim |
| 245 | c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}} |
| 246 | prior := hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c} |
| 247 | if err := w.stopPredecessor(ctx, prior); err != nil || stops != 1 { |
| 248 | t.Fatalf("main's stop: %v (stops %d)", err, stops) |
| 249 | } |
| 250 | if err := w.AfterAdmission(ctx, prior, prior); err != nil || locals != 1 || len(*posted) != 0 { |
| 251 | t.Fatalf("main's handover: %v (locals %d, posted %d)", err, locals, len(*posted)) |
| 252 | } |
| 253 | }) |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | // C10 and N6's Before verify root's receipt, not only find it: a receipt |
| 258 | // in the guard journal whose fence id, phase and digest match the serving |
| 259 | // op's record, but whose bytes disagree with their own fence target, is |
| 260 | // refused by both. With root's genuine receipt both pass. |
| 261 | func TestRelocationCompletedReceiptIsVerifiedByC10AndN6(t *testing.T) { |
| 262 | ctx := t.Context() |
| 263 | for _, genuine := range []bool{true, false} { |
| 264 | t.Run(fmt.Sprintf("genuine %v", genuine), func(t *testing.T) { |
| 265 | f := newStopProbeFixture(t) |
| 266 | if !genuine { |
| 267 | var r StopReceipt |
| 268 | if err := json.Unmarshal(f.receipt, &r); err != nil { |
| 269 | t.Fatal(err) |
| 270 | } |
| 271 | r.BootID = "another-boot" // the target says "boot" |
| 272 | f.receipt = mustMarshalStopReceipt(r) |
| 273 | } |
| 274 | s, err := f.stop() |
| 275 | if err != nil { |
| 276 | t.Fatal(err) |
| 277 | } |
| 278 | prior := f.completedServing(s, false) |
| 279 | check := func(what string, err error) { |
| 280 | t.Helper() |
| 281 | if genuine && err != nil { |
| 282 | t.Fatalf("%s refused root's receipt: %v", what, err) |
| 283 | } |
| 284 | if !genuine && !errors.Is(err, ErrClaimStopReceipt) { |
| 285 | t.Fatalf("%s accepted a receipt that does not verify: %v", what, err) |
| 286 | } |
| 287 | } |
| 288 | h := relocationCompletedHandover{Guard: f.a.Guard, Box: "dst"} |
| 289 | check("N6's Before", h.Before(ctx, prior)) |
| 290 | |
| 291 | // C10 over the persisted binding, the serving claim in the store. |
| 292 | store := release.NewDirStore(t.TempDir()) |
| 293 | key, _ := hostclaim.Key(prior.Host) |
| 294 | if _, err := store.Put(ctx, key, mustJSON(t, prior.Claim), ""); err != nil { |
| 295 | t.Fatal(err) |
| 296 | } |
| 297 | f.a.Claims = store |
| 298 | done, err := f.a.completedRelocationDischarge(ctx, execlease.Binding{Box: "dst", Host: prior.Host, Incarnation: prior.Claim.Holder.Incarnation, Op: "rel-1"}) |
| 299 | if !done { |
| 300 | t.Fatalf("C10 fell through to main's cleanup: %v", err) |
| 301 | } |
| 302 | check("C10", err) |
| 303 | }) |
| 304 | } |
| 305 | } |
| 306 | |
| 307 | // N6 is now always in front of the admission hooks in claim mode, so |
| 308 | // Plan 11's relocating routing reaches its owners through N6's fallback: |
| 309 | // TestPlan11Fix1RelocationAdmissionHooks's three writers, with N6 wired as |
| 310 | // production wires it. |
| 311 | func TestRelocationCompletedHandoverKeepsPlan11Routing(t *testing.T) { |
| 312 | for _, writer := range []string{"source", "", "target"} { |
| 313 | t.Run(writer, func(t *testing.T) { |
| 314 | calls := []string{} |
| 315 | w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { calls = append(calls, "local-stop"); return nil }} |
| 316 | w.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { |
| 317 | calls = append(calls, "local-after") |
| 318 | return nil |
| 319 | } |
| 320 | reloc := ClaimRelocationOwners{BeforeAdmission: func(context.Context, hostclaim.Observed) error { |
| 321 | calls = append(calls, "relocation-before") |
| 322 | return nil |
| 323 | }, AfterAdmission: func(context.Context, hostclaim.Observed, hostclaim.Observed) error { |
| 324 | calls = append(calls, "relocation-after") |
| 325 | return nil |
| 326 | }} |
| 327 | wireProductionAdmission(w, "target", true, claimCrossBoxHandover{}, &reloc, &relocationCompletedHandover{Box: "target"}) |
| 328 | o := plan11Relocating("probing") |
| 329 | o.Claim.Op.Exec = "target" |
| 330 | o.Claim.Holder.WriterBox = writer |
| 331 | if err := w.stopPredecessor(t.Context(), o); err != nil { |
| 332 | t.Fatal(err) |
| 333 | } |
| 334 | if err := w.AfterAdmission(t.Context(), o, o); err != nil { |
| 335 | t.Fatal(err) |
| 336 | } |
| 337 | want := []string{"relocation-before", "relocation-after"} |
| 338 | if writer == "target" { |
| 339 | want = []string{"local-stop", "local-after"} |
| 340 | } |
| 341 | if !slices.Equal(calls, want) { |
| 342 | t.Fatalf("calls=%v want=%v", calls, want) |
| 343 | } |
| 344 | }) |
| 345 | } |
| 346 | } |
| 347 | |
amadan