| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/ed25519" |
| 7 | "encoding/json" |
| 8 | "errors" |
| 9 | "fmt" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "strconv" |
| 13 | "strings" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/hostactors" |
| 18 | "github.com/carlosframework/platform/internal/hostbox" |
| 19 | "github.com/carlosframework/platform/internal/hostclaim" |
| 20 | "github.com/carlosframework/platform/internal/hostevidence" |
| 21 | "github.com/carlosframework/platform/internal/hostguard" |
| 22 | "github.com/carlosframework/platform/internal/hostops" |
| 23 | "github.com/carlosframework/platform/internal/release" |
| 24 | ) |
| 25 | |
| 26 | // receiptWorld is one box's stop path over a signed roster: the production |
| 27 | // roster identity, the box's evidence key bound as its census key by the |
| 28 | // real BindCensusKey writer, a stored claim, the real hostguard fence and |
| 29 | // ClaimStopCoordinator, and receipts published into the deployment store |
| 30 | // the unchanged v0.28.4 console verifiers read. Only the kernel's |
| 31 | // observation and the producer drain are fixtures: they are not the path |
| 32 | // under test. The receipt's numeral and binding are. |
| 33 | type receiptWorld struct { |
| 34 | t *testing.T |
| 35 | roster *bindingRosterFixture |
| 36 | key ed25519.PrivateKey |
| 37 | host string |
| 38 | serving bool |
| 39 | data string |
| 40 | locks string |
| 41 | kernel *receiptKernel |
| 42 | ids int |
| 43 | // installed is the actor whose binding stamps the installation. |
| 44 | installed hostbox.Actor |
| 45 | } |
| 46 | |
| 47 | type receiptKernel struct { |
| 48 | fail error |
| 49 | recovered []hostguard.FenceTarget |
| 50 | } |
| 51 | |
| 52 | func receiptKernelEvidence() StopKernelEvidence { |
| 53 | return StopKernelEvidence{ObservedBootID: "boot-1", InventorySHA256: strings.Repeat("b", 64), LocalJournalSHA256: strings.Repeat("c", 64), CgroupIdentities: []string{"cg-1"}, UnitInvocationIDs: []string{"unit-1"}} |
| 54 | } |
| 55 | |
| 56 | func (k *receiptKernel) VerifyStopped(context.Context, hostguard.FenceTarget, StopInventory, DrainAcknowledgement) (StopKernelEvidence, error) { |
| 57 | if k.fail != nil { |
| 58 | return StopKernelEvidence{}, k.fail |
| 59 | } |
| 60 | return receiptKernelEvidence(), nil |
| 61 | } |
| 62 | |
| 63 | func (k *receiptKernel) RecoverStopped(_ context.Context, t hostguard.FenceTarget, _ StopInventory) (StopKernelEvidence, DrainAcknowledgement, error) { |
| 64 | k.recovered = append(k.recovered, t) |
| 65 | return receiptKernelEvidence(), DrainClaimAcknowledgement(t.FenceID, t.ControllerID, "start-dead", nil), nil |
| 66 | } |
| 67 | |
| 68 | func (k *receiptKernel) VerifyNoProcess(context.Context, hostguard.FenceTarget, StopInventory) (StopKernelEvidence, error) { |
| 69 | e := receiptKernelEvidence() |
| 70 | e.UnitInvocationIDs = []string{} |
| 71 | return e, nil |
| 72 | } |
| 73 | |
| 74 | const receiptWorldHost = "shop.bab.oncarlos.com" |
| 75 | |
| 76 | // newReceiptWorld creates the box, binds its census key, creates the claim |
| 77 | // and holds it at writer session 5: an operation's stop (op purge-1) or, with |
| 78 | // serving, an ordinary serving writer's. |
| 79 | func newReceiptWorld(t *testing.T, serving bool) *receiptWorld { |
| 80 | t.Helper() |
| 81 | ctx := t.Context() |
| 82 | w := &receiptWorld{t: t, roster: newBindingRosterFixture(t, "box-1", "ie"), key: ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize)), host: receiptWorldHost, serving: serving, kernel: &receiptKernel{}} |
| 83 | w.roster.bindCensusKey(w.key) |
| 84 | var err error |
| 85 | if w.installed, err = w.roster.identity().Verify(ctx); err != nil { |
| 86 | t.Fatal(err) |
| 87 | } |
| 88 | now := time.Now().UTC().Truncate(time.Second) |
| 89 | in := hostclaim.NewHolder{Family: "instance", Key: "apps/bab/jam/instances/" + w.host + ".json", Account: "bab", App: "jam", Host: w.host, Region: "ie", RecordRegion: "ie", CreatedAt: now} |
| 90 | body := []byte(`{"schema":1,"host":"` + w.host + `","account":"bab","app":"jam","region":"ie","created_at":"` + now.Format(time.RFC3339) + `"}`) |
| 91 | absent, err := hostclaim.Read(ctx, w.roster.store, w.host) |
| 92 | if err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | o, err := hostclaim.Update(ctx, w.roster.store, absent, hostclaim.BeginCreate(in, body, "create", "console", now)) |
| 96 | if err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | if o, err = hostclaim.Update(ctx, w.roster.store, o, hostclaim.CompleteCreate("create", "box-1")); err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | if _, err = hostclaim.Update(ctx, w.roster.store, o, func(c *hostclaim.Claim) error { |
| 103 | h := c.Holder |
| 104 | h.Session, h.WriterBox, h.Generation = 5, "box-1", "g5" |
| 105 | h.Admissions = []hostclaim.AdmissionRecord{{Region: "ie", Session: 5, Box: "box-1", Generation: "g5", Mode: "fresh"}} |
| 106 | if !serving { |
| 107 | c.Op = hostclaim.Op{Kind: hostclaim.KindMoving, ID: "purge-1", Phase: "stopping", Exec: "box-1"} |
| 108 | } |
| 109 | return nil |
| 110 | }); err != nil { |
| 111 | t.Fatal(err) |
| 112 | } |
| 113 | w.data, w.locks = t.TempDir(), t.TempDir() |
| 114 | if err := os.Mkdir(filepath.Join(w.data, "hostguard"), 0770); err != nil { |
| 115 | t.Fatal(err) |
| 116 | } |
| 117 | if err := w.guard().Activate(ctx, w.host, "box-1", nil); err != nil { |
| 118 | t.Fatal(err) |
| 119 | } |
| 120 | return w |
| 121 | } |
| 122 | |
| 123 | func (w *receiptWorld) claim() hostclaim.Observed { |
| 124 | w.t.Helper() |
| 125 | o, err := hostclaim.Read(w.t.Context(), w.roster.store, w.host) |
| 126 | if err != nil { |
| 127 | w.t.Fatal(err) |
| 128 | } |
| 129 | return o |
| 130 | } |
| 131 | |
| 132 | func (w *receiptWorld) guard() hostguard.Guard { |
| 133 | return hostguard.Guard{DataDir: w.data, LockDir: w.locks, Claims: hostguard.StoreReader{Store: w.roster.store}} |
| 134 | } |
| 135 | |
| 136 | // coordinator is root's stop coordinator as one process builds it; a new |
| 137 | // call is a restarted root over the same persisted fence records. |
| 138 | func (w *receiptWorld) coordinator() ClaimStopCoordinator { |
| 139 | inventory := func(_ context.Context, host string) (StopInventory, error) { |
| 140 | h := w.claim().Claim.Holder |
| 141 | return StopInventory{Host: host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, Box: "box-1", Enrollment: w.installed.Enrollment, DirectoryRevision: w.installed.Binding, Complete: true, Entries: []StopInventoryEntry{{Kind: "path", ArtifactID: "artifact-1", ManagedRoot: "data", RelativePath: "instances/" + host}}}, nil |
| 142 | } |
| 143 | return ClaimStopCoordinator{VerifiedRosterRevision: (&claimCleanupEnrollment{actor: w.installed, directory: w.roster.dir}).rosterRevision, EvidenceSigner: w.key, RequestLimit: hostbox.RequestLimit, Guard: w.guard(), Claims: hostguard.StoreReader{Store: w.roster.store}, CarlosUID: 1001, Evidence: w.roster.store, |
| 144 | Peers: stopPeerFunc(func(context.Context) (StopPeer, error) { |
| 145 | return StopPeer{UID: 1001, PID: 42, ProcessStart: "start-42", BootID: "boot-1", ControllerID: "controller-1"}, nil |
| 146 | }), |
| 147 | Inventory: exactEvidenceInventory{stopInventoryFunc(inventory)}, |
| 148 | Drainer: stopDrainFunc(func(_ context.Context, _ StopRequest, fence string) (DrainAcknowledgement, error) { |
| 149 | return DrainClaimAcknowledgement(fence, "controller-1", "start-42", nil), nil |
| 150 | }), |
| 151 | Kernel: w.kernel, Now: time.Now, |
| 152 | NewID: func() string { w.ids++; return fmt.Sprintf("id-%d", w.ids) }} |
| 153 | } |
| 154 | |
| 155 | // request is the agent's stop request as its producers build it from a |
| 156 | // fresh actor: the installation's binding and the roster counter it read. |
| 157 | func (w *receiptWorld) request(actor hostbox.Actor) StopRequest { |
| 158 | w.t.Helper() |
| 159 | o := w.claim() |
| 160 | h := o.Claim.Holder |
| 161 | r := StopRequest{Region: actor.Region, Host: w.host, Incarnation: h.Incarnation, InstallationStamp: h.Stamp, WriterSession: h.Session, Box: actor.Box, Enrollment: actor.Enrollment, |
| 162 | BootID: "boot-1", ControllerID: "controller-1", ExpectedClaimETag: o.ETag, ExpectedDirectoryRevision: w.installed.Binding, RosterRevision: actor.Revision} |
| 163 | if !w.serving { |
| 164 | r.OperationID, r.Phase, r.ObligationID = "purge-1", "stopping", "pre-teardown-stop" |
| 165 | } |
| 166 | return r |
| 167 | } |
| 168 | |
| 169 | func (w *receiptWorld) fresh() hostbox.Actor { |
| 170 | w.t.Helper() |
| 171 | a, err := w.roster.identity().Verify(w.t.Context()) |
| 172 | if err != nil { |
| 173 | w.t.Fatal(err) |
| 174 | } |
| 175 | return a |
| 176 | } |
| 177 | |
| 178 | func decodeStopReceipt(t *testing.T, raw []byte) StopReceipt { |
| 179 | t.Helper() |
| 180 | var r StopReceipt |
| 181 | if err := json.Unmarshal(raw, &r); err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | return r |
| 185 | } |
| 186 | |
| 187 | // purgeVerify runs the unchanged console's StopBeforeTeardown twice, as the |
| 188 | // purge's first phase does: the first collects and verifies the receipt and |
| 189 | // cites it in the claim (ErrPending), the second verifies the citation. |
| 190 | func (w *receiptWorld) purgeVerify() error { |
| 191 | authority := hostops.ProductionAuthority{Store: w.roster.store, Directory: w.roster.dir, Now: time.Now} |
| 192 | err := authority.StopBeforeTeardown(w.t.Context(), w.claim()) |
| 193 | if !errors.Is(err, hostops.ErrPending) { |
| 194 | return err |
| 195 | } |
| 196 | return authority.StopBeforeTeardown(w.t.Context(), w.claim()) |
| 197 | } |
| 198 | |
| 199 | // wakeResetVerify is the unchanged console's WakeResetTransport.VerifyReceipt |
| 200 | // for this serving session's stop, named at the claim the stop was asked at. |
| 201 | func (w *receiptWorld) wakeResetVerify(stopped hostclaim.Observed) error { |
| 202 | h := stopped.Claim.Holder |
| 203 | now := time.Now().UTC() |
| 204 | r := hostops.WakeResetRequest{Admission: hostops.WakeResetAdmission{Account: "bab", App: "jam", AppID: "app-1", Registration: "registration-1", Epoch: 1, Owner: hostops.RegistrationOwner{Console: "console", Enrollment: strings.Repeat("d", 32), Controller: "console-controller"}}, |
| 205 | V: 1, ID: strings.Repeat("f", 64), Bucket: bindingFixtureBucket, Console: "console", ConsoleEnrollment: strings.Repeat("d", 32), Namespace: "platform", Host: w.host, |
| 206 | ClaimETag: stopped.ETag, ClaimGen: stopped.Claim.Gen, Incarnation: h.Incarnation, Stamp: h.Stamp, Session: h.Session, Episode: "episode-1", |
| 207 | Attempt: hostclaim.WakeRetryAttempt{ID: "attempt-1", Box: "box-1", Enrollment: w.installed.Enrollment, BootID: "boot-1", ControllerID: "controller-1"}, IssuedAt: now, Expires: now.Add(time.Minute)} |
| 208 | return hostops.WakeResetTransport{Store: w.roster.store, Directory: w.roster.dir, Now: time.Now}.VerifyReceipt(w.t.Context(), r) |
| 209 | } |
| 210 | |
| 211 | // L7a (roster binding task 4): the normal and no-process receipt |
| 212 | // constructors take directory_revision from the stop target's roster |
| 213 | // numeral, never the binding, and carry the installation's binding in |
| 214 | // binding. The numeral is the counter of the roster the requester read; a |
| 215 | // retry after roster writes resumes the persisted fence and returns the |
| 216 | // original receipt with its original numeral. |
| 217 | func TestStopReceiptCarriesNumeralAndBinding(t *testing.T) { |
| 218 | ctx := t.Context() |
| 219 | for _, serving := range []bool{false, true} { |
| 220 | t.Run(fmt.Sprintf("serving=%v", serving), func(t *testing.T) { |
| 221 | w := newReceiptWorld(t, serving) |
| 222 | actor := w.fresh() |
| 223 | if !hostactors.ValidBinding(actor.Binding) || actor.Binding != w.installed.Binding { |
| 224 | t.Fatalf("actor binding %q", actor.Binding) |
| 225 | } |
| 226 | req := w.request(actor) |
| 227 | result, err := w.coordinator().FenceAndStop(ctx, req) |
| 228 | if err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | r := decodeStopReceipt(t, result.Bytes) |
| 232 | if r.DirectoryRevision != strconv.FormatUint(actor.Revision, 10) || r.Binding != actor.Binding || result.Target.RosterRevision != actor.Revision || result.Target.DirectoryRevision != actor.Binding { |
| 233 | t.Fatalf("receipt numeral %q binding %q target %+v; want numeral %d binding %s", r.DirectoryRevision, r.Binding, result.Target, actor.Revision, actor.Binding) |
| 234 | } |
| 235 | if _, err := validateTransportStop(req, result); err != nil { |
| 236 | t.Fatal("agent refused the receipt:", err) |
| 237 | } |
| 238 | if err := verifyRetirementReceipt(ctx, hostguard.FenceReceipt{Target: result.Target, Bytes: result.Bytes, EvidenceAt: result.EvidenceAt}); err != nil { |
| 239 | t.Fatal("root's retirement check refused the receipt:", err) |
| 240 | } |
| 241 | |
| 242 | // Two console creates' holds land; the agent retries with the newer |
| 243 | // numeral. Same fence, same receipt, the original numeral. |
| 244 | w.roster.holdChurn() |
| 245 | w.roster.holdChurn() |
| 246 | later := w.fresh() |
| 247 | if later.Revision != actor.Revision+4 || later.Binding != actor.Binding { |
| 248 | t.Fatalf("hold churn moved revision %d -> %d, binding %v", actor.Revision, later.Revision, later.Binding == actor.Binding) |
| 249 | } |
| 250 | retry := w.request(later) |
| 251 | again, err := w.coordinator().FenceAndStop(ctx, retry) |
| 252 | if err != nil { |
| 253 | t.Fatal("retry after hold churn:", err) |
| 254 | } |
| 255 | if !bytes.Equal(again.Bytes, result.Bytes) || again.Target != result.Target { |
| 256 | t.Fatal("retry after hold churn did not return the original receipt") |
| 257 | } |
| 258 | if _, err := validateTransportStop(retry, again); err != nil { |
| 259 | t.Fatal("agent refused the original numeral on retry:", err) |
| 260 | } |
| 261 | }) |
| 262 | } |
| 263 | |
| 264 | // The no-process witness (writer session zero): no fence, the witness's |
| 265 | // own target is the one source. |
| 266 | w := newReceiptWorld(t, true) |
| 267 | o := w.claim() |
| 268 | if _, err := hostclaim.Update(ctx, w.roster.store, o, func(c *hostclaim.Claim) error { |
| 269 | c.Holder.Session, c.Holder.WriterBox, c.Holder.Generation, c.Holder.Admissions = 0, "", "", nil |
| 270 | return nil |
| 271 | }); err != nil { |
| 272 | t.Fatal(err) |
| 273 | } |
| 274 | actor := w.fresh() |
| 275 | req := w.request(actor) |
| 276 | result, err := w.coordinator().FenceAndStop(ctx, req) |
| 277 | if err != nil { |
| 278 | t.Fatal("no-process witness:", err) |
| 279 | } |
| 280 | r := decodeStopReceipt(t, result.Bytes) |
| 281 | if r.Kind != StopReceiptNoProcess || r.DirectoryRevision != strconv.FormatUint(actor.Revision, 10) || r.Binding != actor.Binding || result.Target.RosterRevision != actor.Revision { |
| 282 | t.Fatalf("no-process receipt numeral %q binding %q kind %q", r.DirectoryRevision, r.Binding, r.Kind) |
| 283 | } |
| 284 | if _, err := validateTransportStop(req, result); err != nil { |
| 285 | t.Fatal("agent refused the no-process receipt:", err) |
| 286 | } |
| 287 | if err := validateServingLegacyStop(servingLegacyStop{Observed: o, Result: result}); err == nil { |
| 288 | // o is the claim before the session reset; the stored legacy stop |
| 289 | // names the claim the witness read. |
| 290 | t.Fatal("legacy stop validated against the wrong claim") |
| 291 | } |
| 292 | if err := validateServingLegacyStop(servingLegacyStop{Observed: w.claim(), Result: result}); err != nil { |
| 293 | t.Fatal("retained no-process stop:", err) |
| 294 | } |
| 295 | } |
| 296 | |
| 297 | // N1 (roster binding task 4): the fence persists the requester's roster |
| 298 | // numeral before any stop effect. Root crashes after BeginFence and before |
| 299 | // CompleteFence; holds land; a restarted root's RecoverPending recovers the |
| 300 | // stop from the fence alone. The recovered receipt's directory_revision is |
| 301 | // the persisted numeral and its binding the installation's stamp, and the |
| 302 | // branch's unchanged console verifiers accept it: the purge's |
| 303 | // StopBeforeTeardown for an operation stop, WakeResetTransport.VerifyReceipt |
| 304 | // for a serving stop. A fence persisted without a numeral is refused. |
| 305 | func TestRecoveredReceiptKeepsRosterNumeral(t *testing.T) { |
| 306 | ctx := t.Context() |
| 307 | for _, serving := range []bool{false, true} { |
| 308 | t.Run(fmt.Sprintf("serving=%v", serving), func(t *testing.T) { |
| 309 | w := newReceiptWorld(t, serving) |
| 310 | actor := w.fresh() |
| 311 | req := w.request(actor) |
| 312 | stopped := w.claim() |
| 313 | crash := errors.New("root crashed after the fence") |
| 314 | w.kernel.fail = crash |
| 315 | if _, err := w.coordinator().FenceAndStop(ctx, req); !errors.Is(err, crash) { |
| 316 | t.Fatal(err) |
| 317 | } |
| 318 | pending, err := w.guard().PendingStopFences(ctx) |
| 319 | if err != nil || len(pending) != 1 || pending[0].RosterRevision != actor.Revision || pending[0].DirectoryRevision != actor.Binding { |
| 320 | t.Fatalf("persisted fence %+v %v", pending, err) |
| 321 | } |
| 322 | |
| 323 | w.roster.holdChurn() |
| 324 | w.roster.activationChurn() |
| 325 | w.kernel.fail = nil |
| 326 | if err := (ClaimStopHTTP{Coordinator: w.coordinator()}).RecoverPending(ctx); err != nil { |
| 327 | t.Fatal("restarted root's recovery:", err) |
| 328 | } |
| 329 | if len(w.kernel.recovered) != 1 { |
| 330 | t.Fatalf("recovered %d", len(w.kernel.recovered)) |
| 331 | } |
| 332 | durable, err := w.guard().FenceReceiptFor(ctx, pending[0]) |
| 333 | if err != nil { |
| 334 | t.Fatal(err) |
| 335 | } |
| 336 | r := decodeStopReceipt(t, durable.Bytes) |
| 337 | if r.DirectoryRevision != strconv.FormatUint(actor.Revision, 10) || r.Binding != actor.Binding { |
| 338 | t.Fatalf("recovered receipt numeral %q binding %q; want %d and %s", r.DirectoryRevision, r.Binding, actor.Revision, actor.Binding) |
| 339 | } |
| 340 | |
| 341 | // The agent's next stop request carries the newer numeral and is |
| 342 | // a recovery of the dead controller's tuple: root publishes the |
| 343 | // recovered receipt, unchanged. |
| 344 | later := w.fresh() |
| 345 | if later.Revision != actor.Revision+4 || later.Binding != actor.Binding { |
| 346 | t.Fatalf("churn moved revision %d -> %d", actor.Revision, later.Revision) |
| 347 | } |
| 348 | retry := w.request(later) |
| 349 | retry.Recover = true |
| 350 | result, err := w.coordinator().FenceAndStop(ctx, retry) |
| 351 | if err != nil { |
| 352 | t.Fatal("publication of the recovered receipt:", err) |
| 353 | } |
| 354 | if !bytes.Equal(result.Bytes, durable.Bytes) || result.Target != pending[0] { |
| 355 | t.Fatal("published another receipt than the recovered one") |
| 356 | } |
| 357 | if _, err := validateTransportStop(retry, result); err != nil { |
| 358 | t.Fatal("agent refused the recovered receipt:", err) |
| 359 | } |
| 360 | if serving { |
| 361 | if err := w.wakeResetVerify(stopped); err != nil { |
| 362 | t.Fatal("unchanged wake-reset verifier refused the recovered receipt:", err) |
| 363 | } |
| 364 | } else if err := w.purgeVerify(); err != nil { |
| 365 | t.Fatal("unchanged purge verifier refused the recovered receipt:", err) |
| 366 | } |
| 367 | }) |
| 368 | } |
| 369 | |
| 370 | // A fence persisted before roster binding has no numeral: recovery |
| 371 | // refuses it before any kill and journals nothing. |
| 372 | w := newReceiptWorld(t, false) |
| 373 | actor := w.fresh() |
| 374 | req := w.request(actor) |
| 375 | legacy := hostguard.FenceTarget{FenceID: "legacy", ClaimGen: w.claim().Claim.Gen, Region: req.Region, Host: req.Host, Incarnation: req.Incarnation, Stamp: req.InstallationStamp, Session: req.WriterSession, Op: req.OperationID, Phase: req.Phase, ObligationID: req.ObligationID, |
| 376 | Box: req.Box, Enrollment: req.Enrollment, BootID: req.BootID, ControllerID: req.ControllerID, ClaimETag: req.ExpectedClaimETag, DirectoryRevision: req.ExpectedDirectoryRevision} |
| 377 | if _, err := w.guard().BeginFence(ctx, legacy); err != nil { |
| 378 | t.Fatal(err) |
| 379 | } |
| 380 | if err := (ClaimStopHTTP{Coordinator: w.coordinator()}).RecoverPending(ctx); !errors.Is(err, ErrClaimStopReceipt) || len(w.kernel.recovered) != 0 { |
| 381 | t.Fatalf("legacy fence recovery: %v, kills %d", err, len(w.kernel.recovered)) |
| 382 | } |
| 383 | if _, err := w.guard().FenceReceiptFor(ctx, legacy); !errors.Is(err, hostguard.ErrNoFenceReceipt) { |
| 384 | t.Fatalf("legacy fence journaled a receipt: %v", err) |
| 385 | } |
| 386 | } |
| 387 | |
| 388 | // publishTampered publishes body, re-signed with the box's evidence key, |
| 389 | // under a fresh receipt id: a receipt only a wrong producer would write. |
| 390 | func (w *receiptWorld) publishTampered(result StopResult, edit func(map[string]any)) { |
| 391 | w.t.Helper() |
| 392 | var fields map[string]any |
| 393 | if err := json.Unmarshal(result.Bytes, &fields); err != nil { |
| 394 | w.t.Fatal(err) |
| 395 | } |
| 396 | edit(fields) |
| 397 | fields["id"] = "tampered" |
| 398 | body, err := json.Marshal(fields) |
| 399 | if err != nil { |
| 400 | w.t.Fatal(err) |
| 401 | } |
| 402 | r := decodeStopReceipt(w.t, body) |
| 403 | if err := hostevidence.Publish(w.t.Context(), w.roster.store, w.key, stopEvidenceKey(r.Host, r.OperationID, r.Box, r.ID), body); err != nil { |
| 404 | w.t.Fatal(err) |
| 405 | } |
| 406 | } |
| 407 | |
| 408 | // stopUnpublished collects a genuine receipt through the real coordinator |
| 409 | // without publishing it into the deployment store. |
| 410 | func (w *receiptWorld) stopUnpublished(req StopRequest) StopResult { |
| 411 | w.t.Helper() |
| 412 | c := w.coordinator() |
| 413 | c.Evidence = release.NewDirStore(w.t.TempDir()) |
| 414 | result, err := c.FenceAndStop(w.t.Context(), req) |
| 415 | if err != nil { |
| 416 | w.t.Fatal(err) |
| 417 | } |
| 418 | return result |
| 419 | } |
| 420 | |
| 421 | // L7b (roster binding R1, mixed version): a task 4 receipt, signed with the |
| 422 | // box's evidence key under the operation's evidence/ key, passes the |
| 423 | // branch's unchanged StopBeforeTeardown and verifyEvidence after further |
| 424 | // roster writes. The same receipt with the binding in directory_revision is |
| 425 | // refused. |
| 426 | func TestPurgeAcceptsC1ReceiptOnUnchangedVerifier(t *testing.T) { |
| 427 | w := newReceiptWorld(t, false) |
| 428 | if _, err := w.coordinator().FenceAndStop(t.Context(), w.request(w.fresh())); err != nil { |
| 429 | t.Fatal(err) |
| 430 | } |
| 431 | w.roster.holdChurn() |
| 432 | if err := w.purgeVerify(); err != nil { |
| 433 | t.Fatal("unchanged purge verifier refused a task 4 receipt:", err) |
| 434 | } |
| 435 | |
| 436 | w = newReceiptWorld(t, false) |
| 437 | result := w.stopUnpublished(w.request(w.fresh())) |
| 438 | w.publishTampered(result, func(f map[string]any) { f["directory_revision"] = f["binding"]; delete(f, "binding") }) |
| 439 | if err := w.purgeVerify(); !errors.Is(err, hostops.ErrIdentity) { |
| 440 | t.Fatalf("purge verifier with the binding in directory_revision: %v", err) |
| 441 | } |
| 442 | } |
| 443 | |
| 444 | // L7c (roster binding R1, mixed version): the same through the unchanged |
| 445 | // WakeResetTransport.VerifyReceipt under serving-evidence/, and a |
| 446 | // quarantined actor is still refused before any receipt is read. |
| 447 | func TestWakeResetAcceptsC1ReceiptOnUnchangedVerifier(t *testing.T) { |
| 448 | w := newReceiptWorld(t, true) |
| 449 | stopped := w.claim() |
| 450 | if _, err := w.coordinator().FenceAndStop(t.Context(), w.request(w.fresh())); err != nil { |
| 451 | t.Fatal(err) |
| 452 | } |
| 453 | w.roster.holdChurn() |
| 454 | if err := w.wakeResetVerify(stopped); err != nil { |
| 455 | t.Fatal("unchanged wake-reset verifier refused a task 4 receipt:", err) |
| 456 | } |
| 457 | w.roster.quarantine("loss_1") |
| 458 | if err := w.wakeResetVerify(stopped); !errors.Is(err, hostops.ErrIdentity) { |
| 459 | t.Fatalf("wake reset for a quarantined actor: %v", err) |
| 460 | } |
| 461 | |
| 462 | w = newReceiptWorld(t, true) |
| 463 | stopped = w.claim() |
| 464 | result := w.stopUnpublished(w.request(w.fresh())) |
| 465 | w.publishTampered(result, func(f map[string]any) { f["directory_revision"] = f["binding"]; delete(f, "binding") }) |
| 466 | if err := w.wakeResetVerify(stopped); !errors.Is(err, hostops.ErrWakeResetUncertain) { |
| 467 | t.Fatalf("wake-reset verifier with the binding in directory_revision: %v", err) |
| 468 | } |
| 469 | } |
| 470 | |
amadan