| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "maps" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/hostactors" |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | "github.com/carlosframework/platform/internal/provision" |
| 16 | "github.com/carlosframework/platform/internal/release" |
| 17 | ) |
| 18 | |
| 19 | // resolveControlStore is the ControlWriter's raw store in these tests. Only |
| 20 | // a fencing CAS writes the record through it, so fences counts those. It can |
| 21 | // fail record reads, and run a hook just before the first fence. |
| 22 | type resolveControlStore struct { |
| 23 | release.Store |
| 24 | recordKey string |
| 25 | failGet bool |
| 26 | fences int |
| 27 | beforeFence func() |
| 28 | } |
| 29 | |
| 30 | func (s *resolveControlStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 31 | if key == s.recordKey && s.failGet { |
| 32 | return nil, "", errors.New("record read: connection reset") |
| 33 | } |
| 34 | return s.Store.Get(ctx, key) |
| 35 | } |
| 36 | |
| 37 | func (s *resolveControlStore) Put(ctx context.Context, key string, body []byte, match string) (string, error) { |
| 38 | if key == s.recordKey { |
| 39 | s.fences++ |
| 40 | if f := s.beforeFence; f != nil { |
| 41 | s.beforeFence = nil |
| 42 | f() |
| 43 | } |
| 44 | } |
| 45 | return s.Store.Put(ctx, key, body, match) |
| 46 | } |
| 47 | |
| 48 | // lostRecordPut is the raw store beneath the conditional record writer. Its |
| 49 | // first Put of the record loses its answer: with landed false it fails |
| 50 | // without writing, with landed true it writes and then fails. It keeps that |
| 51 | // Put's body and If-Match ETag, so a test can deliver it late. |
| 52 | type lostRecordPut struct { |
| 53 | release.Store |
| 54 | recordKey string |
| 55 | landed bool |
| 56 | armed bool |
| 57 | body []byte |
| 58 | etag string |
| 59 | } |
| 60 | |
| 61 | func (s *lostRecordPut) Put(ctx context.Context, key string, body []byte, match string) (string, error) { |
| 62 | if key != s.recordKey || !s.armed { |
| 63 | return s.Store.Put(ctx, key, body, match) |
| 64 | } |
| 65 | s.armed = false |
| 66 | s.body, s.etag = append([]byte(nil), body...), match |
| 67 | if s.landed { |
| 68 | if _, err := s.Store.Put(ctx, key, body, match); err != nil { |
| 69 | return "", err |
| 70 | } |
| 71 | } |
| 72 | return "", errors.New("record put: connection reset") |
| 73 | } |
| 74 | |
| 75 | // resolveFixture is the moves door's failure on a claim-on console, through |
| 76 | // the production data handle: RegisterMove, BeginClaimMove, then a |
| 77 | // StartClaimMoveRecord whose record Put loses its answer after it was |
| 78 | // issued, leaving the move-start effect uncertain. |
| 79 | type resolveFixture struct { |
| 80 | *moveStartFixture |
| 81 | control *resolveControlStore |
| 82 | lost *lostRecordPut |
| 83 | data release.Store |
| 84 | h *StartHandle |
| 85 | serving hostclaim.Observed |
| 86 | moving hostclaim.Observed |
| 87 | start provision.MoveStart |
| 88 | before []byte |
| 89 | tag string |
| 90 | } |
| 91 | |
| 92 | func newResolveFixture(t *testing.T, landed bool) *resolveFixture { |
| 93 | t.Helper() |
| 94 | ctx := context.Background() |
| 95 | f := newMoveStartFixture(t, nil) |
| 96 | r := &resolveFixture{moveStartFixture: f} |
| 97 | r.control = &resolveControlStore{Store: f.raw, recordKey: f.raw.recordKey} |
| 98 | var err error |
| 99 | if f.w, err = NewControlWriter(r.control, f.pub, func() time.Time { return f.now }); err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | r.lost = &lostRecordPut{Store: f.raw, recordKey: f.raw.recordKey, landed: landed, armed: true} |
| 103 | records, err := NewConditionalRecordStore(r.lost, f.pub) |
| 104 | if err != nil { |
| 105 | t.Fatal(err) |
| 106 | } |
| 107 | check, err := NewProductionMutationCheck(f.raw, f.w) |
| 108 | if err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | if r.data, err = NewMutationBoundary(true, records, check); err != nil { |
| 112 | t.Fatal(err) |
| 113 | } |
| 114 | f.authority.Store, f.authority.Control = r.data, f.w |
| 115 | f.authority.Directory = hostactors.Directory{Store: r.data, PublicKey: f.pub, Bucket: "bucket"} |
| 116 | if r.before, r.tag, err = f.raw.Store.Get(ctx, f.raw.recordKey); err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | r.serving = f.read(t) |
| 120 | if r.h, err = f.authority.RegisterMove(ctx, r.serving, "canonical", moveStartID); err != nil { |
| 121 | t.Fatal(err) |
| 122 | } |
| 123 | if r.moving, err = BeginClaimMove(ctx, r.data, f.authority.Directory, r.h, r.serving, moveStartID, "operator", "uk-box", "uk", f.now); err != nil { |
| 124 | t.Fatal(err) |
| 125 | } |
| 126 | r.start = provision.MoveStart{ID: moveStartID, Incarnation: f.inc, From: "ie", To: "uk", Cut: "auto", By: "operator", Now: f.now} |
| 127 | if _, err = StartClaimMoveRecord(ctx, r.data, r.h, r.moving, r.start); err == nil { |
| 128 | t.Fatal("the record start succeeded through a lost Put") |
| 129 | } |
| 130 | if r.lost.armed || r.lost.etag != r.tag { |
| 131 | t.Fatalf("the start's record Put matched on %q, want the record's ETag %q", r.lost.etag, r.tag) |
| 132 | } |
| 133 | if e := r.effect(t); e.State != "uncertain" || e.Action != moveStartAction { |
| 134 | t.Fatalf("move-start effect after the lost Put: %+v", e) |
| 135 | } |
| 136 | return r |
| 137 | } |
| 138 | |
| 139 | func (r *resolveFixture) effect(t *testing.T) Effect { |
| 140 | t.Helper() |
| 141 | a, err := ReadAdmission(context.Background(), r.raw.Store, "bab", "app", "canonical") |
| 142 | if err != nil { |
| 143 | t.Fatal(err) |
| 144 | } |
| 145 | return a.Value.Effects[moveStartID] |
| 146 | } |
| 147 | |
| 148 | func (r *resolveFixture) record(t *testing.T) ([]byte, string) { |
| 149 | t.Helper() |
| 150 | body, tag, err := r.raw.Store.Get(context.Background(), r.raw.recordKey) |
| 151 | if err != nil { |
| 152 | t.Fatal(err) |
| 153 | } |
| 154 | return body, tag |
| 155 | } |
| 156 | |
| 157 | // latePut delivers the lost start Put now, with its original If-Match ETag. |
| 158 | func (r *resolveFixture) latePut() error { |
| 159 | _, err := r.raw.Store.Put(context.Background(), r.raw.recordKey, r.lost.body, r.lost.etag) |
| 160 | return err |
| 161 | } |
| 162 | |
| 163 | // A Put that failed before landing: the record lacks the journal entry and |
| 164 | // still carries the ETag the Put matched on, so a fencing CAS rewrites the |
| 165 | // same record value under a new ETag. The late Put can then only fail. |
| 166 | func TestPlan9B8AcceptancePutLostBeforeLandingIsFencedNotApplied(t *testing.T) { |
| 167 | ctx := context.Background() |
| 168 | r := newResolveFixture(t, false) |
| 169 | got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID) |
| 170 | if err != nil || got != OutcomeNotApplied { |
| 171 | t.Fatalf("ResolveMoveStart = %q, %v; want not-applied", got, err) |
| 172 | } |
| 173 | if r.control.fences != 1 { |
| 174 | t.Fatalf("fencing CAS writes = %d, want 1", r.control.fences) |
| 175 | } |
| 176 | body, tag := r.record(t) |
| 177 | if tag == r.tag { |
| 178 | t.Fatal("the fence left the ETag the start Put matched on") |
| 179 | } |
| 180 | before, _, _ := recordSequence(r.before) |
| 181 | after, seq, parseErr := recordSequence(body) |
| 182 | delete(before, "record_seq") |
| 183 | delete(after, "record_seq") |
| 184 | if parseErr != nil || seq != 1 || !maps.EqualFunc(before, after, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { |
| 185 | t.Fatalf("fence did not preserve the record and advance sequence: %s %v", body, parseErr) |
| 186 | } |
| 187 | if rec, err := provision.DecodeRecord(body); err != nil || rec.Move != nil { |
| 188 | t.Fatalf("fenced record: %+v %v", rec.Move, err) |
| 189 | } |
| 190 | if err := r.latePut(); !errors.Is(err, release.ErrCASConflict) { |
| 191 | t.Fatalf("the late start Put after the fence: %v", err) |
| 192 | } |
| 193 | if e := r.effect(t); e.State != "complete" || e.ResultReference != "not-applied" { |
| 194 | t.Fatalf("effect = %+v, want complete not-applied", e) |
| 195 | } |
| 196 | // Asking again answers the recorded outcome and writes nothing. |
| 197 | if got, err = ResolveMoveStart(ctx, r.h, r.moving, moveStartID); err != nil || got != OutcomeNotApplied || r.control.fences != 1 { |
| 198 | t.Fatalf("second ResolveMoveStart = %q, %v, fences %d", got, err, r.control.fences) |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | // A Put that landed but whose response was lost: the record carries this |
| 203 | // move's journal entry, so the effect is complete and nothing is written. |
| 204 | func TestPlan9B8PutLandedResponseLostIsComplete(t *testing.T) { |
| 205 | ctx := context.Background() |
| 206 | r := newResolveFixture(t, true) |
| 207 | _, landed := r.record(t) |
| 208 | got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID) |
| 209 | if err != nil || got != OutcomeComplete { |
| 210 | t.Fatalf("ResolveMoveStart = %q, %v; want complete", got, err) |
| 211 | } |
| 212 | if _, tag := r.record(t); r.control.fences != 0 || tag != landed { |
| 213 | t.Fatalf("a landed start was fenced: fences %d", r.control.fences) |
| 214 | } |
| 215 | if e := r.effect(t); e.State != "complete" || e.ResultReference != moveStartID { |
| 216 | t.Fatalf("effect = %+v, want complete with the move id", e) |
| 217 | } |
| 218 | // The start's replay reads the journaled record and writes nothing. |
| 219 | rec, err := StartClaimMoveRecord(ctx, r.data, r.h, r.moving, r.start) |
| 220 | if err != nil || rec.Move == nil || rec.Move.ID != moveStartID { |
| 221 | t.Fatalf("replay = %+v, %v", rec, err) |
| 222 | } |
| 223 | if _, tag := r.record(t); tag != landed { |
| 224 | t.Fatal("the replay wrote the record") |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | // A late Put that lands between the read and the fence makes the fencing |
| 229 | // CAS lose; the reread finds the journal entry and the effect is complete. |
| 230 | func TestPlan9B8AcceptanceLatePutBeforeTheFenceIsComplete(t *testing.T) { |
| 231 | ctx := context.Background() |
| 232 | r := newResolveFixture(t, false) |
| 233 | r.control.beforeFence = func() { |
| 234 | if err := r.latePut(); err != nil { |
| 235 | t.Error(err) |
| 236 | } |
| 237 | } |
| 238 | got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID) |
| 239 | if err != nil || got != OutcomeComplete { |
| 240 | t.Fatalf("ResolveMoveStart = %q, %v; want complete", got, err) |
| 241 | } |
| 242 | if r.control.fences != 1 { |
| 243 | t.Fatalf("fencing CAS writes = %d, want the one that lost", r.control.fences) |
| 244 | } |
| 245 | body, _ := r.record(t) |
| 246 | if !bytes.Equal(body, r.lost.body) { |
| 247 | t.Fatal("the losing fence wrote the record") |
| 248 | } |
| 249 | if e := r.effect(t); e.State != "complete" || e.ResultReference != moveStartID { |
| 250 | t.Fatalf("effect = %+v, want complete with the move id", e) |
| 251 | } |
| 252 | } |
| 253 | |
| 254 | // A concurrent unrelated write already moved the record off the ETag the |
| 255 | // failed Put matched on: that Put is fenced, and no fencing write is made. |
| 256 | func TestPlan9B8AcceptanceUnrelatedWriteIsAlreadyAFence(t *testing.T) { |
| 257 | ctx := context.Background() |
| 258 | r := newResolveFixture(t, false) |
| 259 | var rec provision.Record |
| 260 | if err := json.Unmarshal(r.before, &rec); err != nil { |
| 261 | t.Fatal(err) |
| 262 | } |
| 263 | rec.Channel = "beta" |
| 264 | rec.RecordSeq = 1 // An unrelated claim-on writer advanced the counter. |
| 265 | body, err := json.Marshal(rec) |
| 266 | if err != nil { |
| 267 | t.Fatal(err) |
| 268 | } |
| 269 | moved, err := r.raw.Store.Put(ctx, r.raw.recordKey, body, r.tag) |
| 270 | if err != nil { |
| 271 | t.Fatal(err) |
| 272 | } |
| 273 | got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID) |
| 274 | if err != nil || got != OutcomeNotApplied { |
| 275 | t.Fatalf("ResolveMoveStart = %q, %v; want not-applied", got, err) |
| 276 | } |
| 277 | if _, tag := r.record(t); r.control.fences != 0 || tag != moved { |
| 278 | t.Fatalf("an already fenced Put was fenced again: fences %d", r.control.fences) |
| 279 | } |
| 280 | if err := r.latePut(); !errors.Is(err, release.ErrCASConflict) { |
| 281 | t.Fatalf("the late start Put after the unrelated write: %v", err) |
| 282 | } |
| 283 | if e := r.effect(t); e.State != "complete" || e.ResultReference != "not-applied" { |
| 284 | t.Fatalf("effect = %+v, want complete not-applied", e) |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | // A record read error decides nothing: the effect stays uncertain and |
| 289 | // nothing is written. Once the read works, the same handle resolves it. |
| 290 | func TestPlan9B8ReadErrorIsUncertain(t *testing.T) { |
| 291 | ctx := context.Background() |
| 292 | r := newResolveFixture(t, false) |
| 293 | r.control.failGet = true |
| 294 | got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID) |
| 295 | if err == nil || got != OutcomeUncertain { |
| 296 | t.Fatalf("ResolveMoveStart = %q, %v; want uncertain with an error", got, err) |
| 297 | } |
| 298 | if _, tag := r.record(t); r.control.fences != 0 || tag != r.tag { |
| 299 | t.Fatal("an unread record was fenced") |
| 300 | } |
| 301 | if e := r.effect(t); e.State != "uncertain" { |
| 302 | t.Fatalf("effect = %+v, want uncertain", e) |
| 303 | } |
| 304 | r.control.failGet = false |
| 305 | if got, err = ResolveMoveStart(ctx, r.h, r.moving, moveStartID); err != nil || got != OutcomeNotApplied { |
| 306 | t.Fatalf("resolution after read recovery: %q, %v", got, err) |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | // Only the handle whose effect issued the Puts resolves it, for its own |
| 311 | // move. Knowing the registration's id and owner tuple is not enough. |
| 312 | func TestPlan9B8RefusesAnotherHandleOrMove(t *testing.T) { |
| 313 | ctx := context.Background() |
| 314 | r := newResolveFixture(t, false) |
| 315 | otherApp := r.moving |
| 316 | claim := *otherApp.Claim |
| 317 | claim.Op.Intent = maps.Clone(claim.Op.Intent) |
| 318 | claim.Op.Intent["app_id"] = "other" |
| 319 | otherApp.Claim = &claim |
| 320 | forged := &StartHandle{w: r.h.w, account: "bab", app: "app", appID: "canonical", id: moveStartID, owner: r.h.Owner(), epoch: r.h.Epoch()} |
| 321 | for _, tc := range []struct { |
| 322 | name string |
| 323 | h *StartHandle |
| 324 | o hostclaim.Observed |
| 325 | id string |
| 326 | }{ |
| 327 | {"another move id", r.h, r.moving, "mv_" + strings.Repeat("b", 26)}, |
| 328 | {"same owner, no issue", forged, r.moving, moveStartID}, |
| 329 | {"nil handle", nil, r.moving, moveStartID}, |
| 330 | {"serving observation", r.h, r.serving, moveStartID}, |
| 331 | {"another app id", r.h, otherApp, moveStartID}, |
| 332 | {"absent observation", r.h, hostclaim.Observed{Host: r.host, Absent: true}, moveStartID}, |
| 333 | {"empty move id", r.h, r.moving, ""}, |
| 334 | {"another registration", &StartHandle{w: r.h.w}, r.moving, moveStartID}, |
| 335 | {"handle without writer", &StartHandle{id: moveStartID}, r.moving, moveStartID}, |
| 336 | } { |
| 337 | if got, err := ResolveMoveStart(ctx, tc.h, tc.o, tc.id); got != OutcomeUncertain || !errors.Is(err, ErrIdentity) { |
| 338 | t.Errorf("%s: %q, %v; want uncertain ErrIdentity", tc.name, got, err) |
| 339 | } |
| 340 | } |
| 341 | if _, tag := r.record(t); r.control.fences != 0 || tag != r.tag { |
| 342 | t.Fatal("a refused resolution wrote the record") |
| 343 | } |
| 344 | if e := r.effect(t); e.State != "uncertain" { |
| 345 | t.Fatalf("a refused resolution settled the effect: %+v", e) |
| 346 | } |
| 347 | if got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID); err != nil || got != OutcomeNotApplied { |
| 348 | t.Fatalf("issuing handle resolution: %q, %v", got, err) |
| 349 | } |
| 350 | } |
| 351 | |
| 352 | // The runner's settle hook is scoped to an uncertain move-start effect of |
| 353 | // its own registration: another action and an exclusive runner refuse, and |
| 354 | // an effect still issued (its call not joined) is never resolved. |
| 355 | func TestPlan9B8SettleHookIsScopedToAnUncertainMoveStart(t *testing.T) { |
| 356 | ctx := context.Background() |
| 357 | r := newResolveFixture(t, false) |
| 358 | runner := r.h.Runner(r.h.w.now) |
| 359 | e := r.effect(t) |
| 360 | other := e |
| 361 | other.Action = "move-relocating-decision" |
| 362 | if err := runner.resolveUncertain(ctx, other, "not-applied"); !errors.Is(err, ErrContract) { |
| 363 | t.Fatalf("another action: %v", err) |
| 364 | } |
| 365 | exclusive := runner |
| 366 | exclusive.exclusive = true |
| 367 | if err := exclusive.resolveUncertain(ctx, e, "not-applied"); !errors.Is(err, ErrContract) { |
| 368 | t.Fatalf("an exclusive runner: %v", err) |
| 369 | } |
| 370 | issued := Effect{ID: "issued-effect", Resource: "apps/bab/app/other.json", Action: moveStartAction, AppID: "canonical", OperationID: moveStartID, Owner: r.h.Owner()} |
| 371 | _, settle, err := runner.Issue(ctx, issued) |
| 372 | if err != nil { |
| 373 | t.Fatal(err) |
| 374 | } |
| 375 | if err := runner.resolveUncertain(ctx, issued, "not-applied"); !errors.Is(err, ErrPending) { |
| 376 | t.Fatalf("resolved an issued effect: %v", err) |
| 377 | } |
| 378 | _ = settle("", errors.Join(ErrNotApplied, errors.New("never called"))) |
| 379 | if got := r.effect(t); got.State != "uncertain" { |
| 380 | t.Fatalf("the refusals moved the move-start effect: %+v", got) |
| 381 | } |
| 382 | } |
| 383 | |
| 384 | // The moves door's acceptance scenario (moves-phase1-2 26d9e4d1, |
| 385 | // TestMoveDoorHostClaimRetryAfterAFailedRecordStart) in hostops: after a |
| 386 | // not-applied resolution the door releases the claim under the |
| 387 | // registration's grant, the registration drains, and a retry with a new |
| 388 | // move id starts the move. |
| 389 | func TestPlan9B8AcceptanceNotAppliedDrainsAndARetryStarts(t *testing.T) { |
| 390 | ctx := context.Background() |
| 391 | r := newResolveFixture(t, false) |
| 392 | if got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID); err != nil || got != OutcomeNotApplied { |
| 393 | t.Fatalf("ResolveMoveStart = %q, %v", got, err) |
| 394 | } |
| 395 | releaseMove := func(c *hostclaim.Claim) error { |
| 396 | if err := hostclaim.RequireMoveOp(moveStartID, hostclaim.KindMoving, hostclaim.PhaseMoving)(c); err != nil { |
| 397 | return err |
| 398 | } |
| 399 | return hostclaim.ReleaseMove(moveStartID, "ie-box")(c) |
| 400 | } |
| 401 | if _, err := hostclaim.Update(r.h.Lifecycle(ctx), r.data, r.moving, releaseMove); err != nil { |
| 402 | t.Fatalf("the door's claim release: %v", err) |
| 403 | } |
| 404 | if err := r.w.SettleRegistration(ctx, "bab", "app", "canonical", moveStartID, r.now.Add(RegistrationTTL+2*RegistrationSkew)); err != nil { |
| 405 | t.Fatalf("the failed attempt's registration does not drain: %v", err) |
| 406 | } |
| 407 | a, err := ReadAdmission(ctx, r.raw.Store, "bab", "app", "canonical") |
| 408 | if err != nil { |
| 409 | t.Fatal(err) |
| 410 | } |
| 411 | if _, ok := a.Value.Registrations[moveStartID]; ok { |
| 412 | t.Fatal("the failed attempt's registration is still held") |
| 413 | } |
| 414 | retry := "mv_" + strings.Repeat("b", 26) |
| 415 | o := r.read(t) |
| 416 | h, err := r.authority.RegisterMove(ctx, o, "canonical", retry) |
| 417 | if err != nil { |
| 418 | t.Fatal(err) |
| 419 | } |
| 420 | moving, err := BeginClaimMove(ctx, r.data, r.authority.Directory, h, o, retry, "operator", "uk-box", "uk", r.now) |
| 421 | if err != nil { |
| 422 | t.Fatalf("retry start CAS: %v", err) |
| 423 | } |
| 424 | start := r.start |
| 425 | start.ID = retry |
| 426 | rec, err := StartClaimMoveRecord(ctx, r.data, h, moving, start) |
| 427 | if err != nil || rec.Move == nil || rec.Move.ID != retry || rec.Move.State != provision.MovePreparing { |
| 428 | t.Fatalf("retry record start = %+v, %v", rec, err) |
| 429 | } |
| 430 | if a, err = ReadAdmission(ctx, r.raw.Store, "bab", "app", "canonical"); err != nil { |
| 431 | t.Fatal(err) |
| 432 | } |
| 433 | if e := a.Value.Effects[retry]; e.State != "complete" || e.ResultReference != retry { |
| 434 | t.Fatalf("retry effect = %+v", e) |
| 435 | } |
| 436 | if e := a.Value.Effects[moveStartID]; e.State != "complete" || e.ResultReference != "not-applied" { |
| 437 | t.Fatalf("failed attempt's effect = %+v", e) |
| 438 | } |
| 439 | } |
| 440 | |
| 441 | // Why the fence rewrites the value with different bytes: the DirStore's |
| 442 | // ETag, like a single-part S3 ETag, is a content hash, so a byte-identical |
| 443 | // rewrite keeps the ETag a late Put matched on. |
| 444 | func TestPlan9B8IdenticalBytesKeepAContentETag(t *testing.T) { |
| 445 | ctx := context.Background() |
| 446 | s := release.NewDirStore(t.TempDir()) |
| 447 | key := provision.RecordKey("bab", "app", "shop.example.com") |
| 448 | tag, err := s.Put(ctx, key, []byte(`{"schema":1}`), "") |
| 449 | if err != nil { |
| 450 | t.Fatal(err) |
| 451 | } |
| 452 | same, err := s.Put(ctx, key, []byte(`{"schema":1}`), tag) |
| 453 | if err != nil || same != tag { |
| 454 | t.Fatalf("identical rewrite: %q, %v (was %q)", same, err, tag) |
| 455 | } |
| 456 | moved, err := s.Put(ctx, key, []byte("{\"schema\":1}\n"), same) |
| 457 | if err != nil || moved == same { |
| 458 | t.Fatalf("newline rewrite: %q, %v", moved, err) |
| 459 | } |
| 460 | } |
| 461 | |
| 462 | // A changed ETag without a sequence increment is not proof. The resolver |
| 463 | // must perform its own sequence CAS before acknowledging not-applied. |
| 464 | func TestPlan9B8RefusesUnprovenContentFence(t *testing.T) { |
| 465 | ctx := context.Background() |
| 466 | r := newResolveFixture(t, false) |
| 467 | if _, err := r.raw.Store.Put(ctx, r.raw.recordKey, append(r.before, '\n'), r.tag); err != nil { |
| 468 | t.Fatal(err) |
| 469 | } |
| 470 | if got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID); err != nil || got != OutcomeNotApplied || r.control.fences != 1 { |
| 471 | t.Fatalf("content-only change was accepted without a sequence fence: %q %v fences=%d", got, err, r.control.fences) |
| 472 | } |
| 473 | r.ordinaryRewrite(t) |
| 474 | if err := r.latePut(); !errors.Is(err, release.ErrCASConflict) { |
| 475 | t.Fatalf("late Put: %v", err) |
| 476 | } |
| 477 | } |
| 478 | |
| 479 | // ordinaryRewrite releases the claim with the existing grant, then exercises |
| 480 | // the real ordinary record writer and claim-on conditional store. Its |
| 481 | // attempted semantic value is exactly the original record's value. |
| 482 | func (r *resolveFixture) ordinaryRewrite(t *testing.T) { |
| 483 | t.Helper() |
| 484 | ctx := context.Background() |
| 485 | if _, err := hostclaim.Update(r.h.Lifecycle(ctx), r.data, r.moving, hostclaim.ReleaseMove(moveStartID, "ie-box")); err != nil { |
| 486 | t.Fatal(err) |
| 487 | } |
| 488 | records, err := NewConditionalRecordStore(r.raw, r.pub) |
| 489 | if err != nil { |
| 490 | t.Fatal(err) |
| 491 | } |
| 492 | if err = provision.SetRecordRegion(ctx, records, "bab", "app", r.host, "ie"); err != nil { |
| 493 | t.Fatal(err) |
| 494 | } |
| 495 | } |
| 496 | |
| 497 | func TestPlan9B8AcceptanceOrdinaryRewriteCannotReopenFenceABA(t *testing.T) { |
| 498 | ctx := context.Background() |
| 499 | r := newResolveFixture(t, false) |
| 500 | if got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID); err != nil || got != OutcomeNotApplied { |
| 501 | t.Fatalf("resolve: %q %v", got, err) |
| 502 | } |
| 503 | r.ordinaryRewrite(t) |
| 504 | body, tag := r.record(t) |
| 505 | fields, seq, err := recordSequence(body) |
| 506 | original, _, _ := recordSequence(r.before) |
| 507 | delete(fields, "record_seq") |
| 508 | delete(original, "record_seq") |
| 509 | if err != nil || seq != 2 || tag == r.tag || !maps.EqualFunc(fields, original, func(a, b json.RawMessage) bool { return bytes.Equal(a, b) }) { |
| 510 | t.Fatalf("ordinary rewrite restored conditional identity: seq=%d tag=%q body=%s err=%v", seq, tag, body, err) |
| 511 | } |
| 512 | if err := r.latePut(); !errors.Is(err, release.ErrCASConflict) { |
| 513 | t.Fatalf("delayed Put survived fence and ordinary rewrite: %v", err) |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | func TestPlan9B8AcceptanceBoxRewriteFencesWithoutClaimContext(t *testing.T) { |
| 518 | ctx := context.Background() |
| 519 | r := newResolveFixture(t, false) |
| 520 | // First fence introduces the sequence; the next box write must advance it |
| 521 | // even without WithRecordSequence or a console conditional-record wrapper. |
| 522 | if got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID); err != nil || got != OutcomeNotApplied { |
| 523 | t.Fatalf("resolve: %q %v", got, err) |
| 524 | } |
| 525 | if _, err := provision.SetRecordMove(ctx, r.raw.Store, "bab", "app", r.host, func(*provision.Record) error { return nil }); err != nil { |
| 526 | t.Fatal(err) |
| 527 | } |
| 528 | body, _ := r.record(t) |
| 529 | _, seq, err := recordSequence(body) |
| 530 | if err != nil || seq != 2 { |
| 531 | t.Fatalf("box seq=%d %v", seq, err) |
| 532 | } |
| 533 | if err := r.latePut(); !errors.Is(err, release.ErrCASConflict) { |
| 534 | t.Fatalf("box reopened fence: %v", err) |
| 535 | } |
| 536 | } |
| 537 | |
| 538 | func TestPlan9B8AcceptanceBoxAdvanceProvesNotApplied(t *testing.T) { |
| 539 | ctx := context.Background() |
| 540 | r := newResolveFixture(t, false) |
| 541 | if _, err := provision.SetRecordMove(provision.WithRecordSequence(ctx), r.raw.Store, "bab", "app", r.host, func(*provision.Record) error { return nil }); err != nil { |
| 542 | t.Fatal(err) |
| 543 | } |
| 544 | if got, err := ResolveMoveStart(ctx, r.h, r.moving, moveStartID); err != nil || got != OutcomeNotApplied || r.control.fences != 0 { |
| 545 | t.Fatalf("box proof: %q %v fences=%d", got, err, r.control.fences) |
| 546 | } |
| 547 | if err := r.latePut(); !errors.Is(err, release.ErrCASConflict) { |
| 548 | t.Fatalf("late Put: %v", err) |
| 549 | } |
| 550 | } |
| 551 | |
amadan