| new file mode 100644 |
| index 000000000..0a2359bc3 |
| --- /dev/null |
| +++ b/internal/hostclaim/rev713_test.go |
| @@ -0,0 +1,402 @@ |
| +package hostclaim |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "reflect" |
| + "testing" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +func TestAdmitWriterWarmReturnRegion(t *testing.T) { |
| + for _, region := range []string{"ie", ""} { |
| + t.Run("region="+region, func(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + o := servingClaim(t, ctx, s) |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "uk", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "GT"})) |
| + target := o.Claim.Holder.Admissions[0] |
| + o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| + c.Holder.Generation = "GS" |
| + c.Op = Op{Kind: KindReverting, ID: "move", Phase: "return", Exec: "boxS"} |
| + return nil |
| + }) |
| + next, err := Update(ctx, s, o, AdmitWriter(Admission{ |
| + ObservedOpID: "move", ObservedPhase: "return", ObservedSession: target.Session, ObservedGeneration: "GS", |
| + Box: "boxS", Mode: "warm", Region: region, PredecessorRegion: "ignored", |
| + PredecessorCertified: true, Coverage: "clean-return", |
| + })) |
| + if region == "" { |
| + if !errors.Is(err, ErrWrongState) { |
| + t.Fatalf("empty region admission = %v", err) |
| + } |
| + assertAdmissionStored(t, ctx, s, o) |
| + return |
| + } |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + last := next.Claim.Holder.Admissions[1] |
| + if last.Region != "ie" || last.PredecessorRegion != "uk" || last.Predecessor != "GT" || last.Generation != "GS" || last.Covered { |
| + t.Fatalf("warm return = %+v", last) |
| + } |
| + assertAdmissionStored(t, ctx, s, next) |
| + }) |
| + } |
| +} |
| + |
| +func TestAdmitWriterPendingParkCoverage(t *testing.T) { |
| + for _, name := range []string{"whole predecessor region", "no park", "other region", "generation only", "local only"} { |
| + t.Run(name, func(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + o := servingClaim(t, ctx, s) |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "uk", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "GT"})) |
| + o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| + c.Op = Op{Kind: KindReverting, ID: "move", Phase: "return", Exec: "boxS"} |
| + ob := Obligation{ID: "park", Op: "release", Kind: "park-replica", Region: "uk", Owner: "region:uk", Retirement: "ret"} |
| + switch name { |
| + case "no park": |
| + return nil |
| + case "other region": |
| + ob.Region, ob.Owner = "ie", "region:ie" |
| + case "generation only": |
| + ob.Kind, ob.Generation = "park-generation", "GT" |
| + case "local only": |
| + ob.Kind, ob.Region, ob.Owner, ob.Box = "park-local", "", "box:boxT", "boxT" |
| + } |
| + c.Obligations = []Obligation{ob} |
| + return nil |
| + }) |
| + before := o |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{ |
| + ObservedOpID: "move", ObservedPhase: "return", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "GT", |
| + Box: "boxS", Mode: "cold", Region: "ie", NewGeneration: "GS2", |
| + })) |
| + last := o.Claim.Holder.Admissions[1] |
| + if last.Covered != (name == "whole predecessor region") || last.PredecessorRegion != "uk" || last.WritesLost == nil { |
| + t.Fatalf("return admission = %+v", last) |
| + } |
| + if !reflect.DeepEqual(before.Claim.Obligations, o.Claim.Obligations) { |
| + t.Fatal("admission changed park obligations") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestPublishGenerationServingCyclesBoundAdmissions(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + o := servingClaim(t, ctx, s) |
| + newID := counter() |
| + for cycle := 1; cycle <= 20; cycle++ { |
| + generation := fmt.Sprintf("G%d", cycle) |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{ |
| + Region: "ie", PredecessorRegion: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, |
| + Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: generation, |
| + })) |
| + o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, generation, "ie", newID)) |
| + for _, record := range o.Claim.Holder.Admissions { |
| + if !record.Covered { |
| + t.Fatalf("cycle %d left superseded predecessor uncovered: %+v", cycle, record) |
| + } |
| + } |
| + if len(o.Claim.Obligations) != 1 { |
| + t.Fatalf("cycle %d chores = %+v", cycle, o.Claim.Obligations) |
| + } |
| + predecessor := RootGeneration |
| + if cycle > 1 { |
| + predecessor = fmt.Sprintf("G%d", cycle-1) |
| + } |
| + ob := o.Claim.Obligations[0] |
| + if ob.Generation != predecessor { |
| + t.Fatalf("retiring %q, want %q", ob.Generation, predecessor) |
| + } |
| + var err error |
| + o, err = Update(ctx, s, o, CompleteObligation(ob.Op, ob.ID)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + o, err = Read(ctx, s, testHost) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{ |
| + Region: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: generation, |
| + Box: o.Claim.Op.Exec, Mode: "warm", PredecessorCertified: true, |
| + })) |
| + o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, generation, "ie", newID)) |
| + records := o.Claim.Holder.Admissions |
| + if records[len(records)-1].Covered || len(o.Claim.Obligations) != 0 { |
| + t.Fatalf("cycle %d covered or retired live predecessor", cycle) |
| + } |
| + if len(records) > 8 { |
| + t.Fatalf("cycle %d admissions grew to %d", cycle, len(records)) |
| + } |
| + } |
| +} |
| + |
| +func TestPublishGenerationReturnedSourceRetiredLater(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + o := servingClaim(t, ctx, s) |
| + o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| + c.Holder.Generation = "GS" |
| + c.Holder.Admissions = []AdmissionRecord{ |
| + {Region: "uk", Generation: "GT", Predecessor: "GS", PredecessorRegion: "ie"}, |
| + {Region: "ie", Generation: "GS", Predecessor: "GT", PredecessorRegion: "uk"}, |
| + } |
| + return nil |
| + }) |
| + o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "GS", "ie", counter())) |
| + if o.Claim.Holder.Admissions[0].Covered || o.Claim.Holder.Admissions[1].Covered || len(o.Claim.Obligations) != 0 { |
| + t.Fatal("source publication covered a live or foreign predecessor") |
| + } |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{ |
| + Region: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "GS", |
| + Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "GS2", |
| + })) |
| + o = updateGenerationTest(t, ctx, s, o, PublishGeneration("", "", o.Claim.Holder.Session, "GS2", "ie", counter())) |
| + if !o.Claim.Holder.Admissions[0].Covered || o.Claim.Holder.Admissions[1].Covered || !o.Claim.Holder.Admissions[2].Covered || len(o.Claim.Obligations) != 1 || o.Claim.Obligations[0].Generation != "GS" { |
| + t.Fatalf("source retirement after return = %+v; admissions = %+v", o.Claim.Obligations, o.Claim.Holder.Admissions) |
| + } |
| +} |
| + |
| +func TestCoverParkedPredecessors(t *testing.T) { |
| + for _, kind := range []string{KindServing, KindParking} { |
| + t.Run(kind, func(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + o := servingClaim(t, ctx, s) |
| + o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| + if kind == KindParking { |
| + c.Op = Op{Kind: kind, ID: "sweep", Phase: "park", Intent: map[string]string{"park_sweep": "v1"}} |
| + } |
| + c.Holder.Admissions = []AdmissionRecord{ |
| + {Region: "ie", PredecessorRegion: "ie", Predecessor: "G1"}, |
| + {Region: "uk", PredecessorRegion: "ie", Predecessor: ""}, |
| + {Region: "ie", PredecessorRegion: "ie", Predecessor: RootGeneration}, |
| + {Region: "ie", PredecessorRegion: "uk", Predecessor: "G1"}, |
| + {Region: "ie", PredecessorRegion: "ie", Predecessor: "G2"}, |
| + {Region: "ie", Predecessor: "G1"}, |
| + {Region: "ie", PredecessorRegion: "ie", Predecessor: "G3", Covered: true}, |
| + } |
| + c.Obligations = []Obligation{{ID: "existing", Op: "old", Kind: "park-generation", Region: "ie", Generation: "G1", Retirement: "ret", Owner: "region:ie"}} |
| + return nil |
| + }) |
| + before := o |
| + for _, generations := range [][]string{nil, {}, {""}} { |
| + o = updateGenerationTest(t, ctx, s, o, CoverParkedPredecessors(o.Claim.Op.ID, o.Claim.Op.Phase, "ie", generations)) |
| + if !reflect.DeepEqual(o.Claim.Holder, before.Claim.Holder) { |
| + t.Fatalf("empty selection changed records: %+v", o.Claim.Holder.Admissions) |
| + } |
| + } |
| + want := *o.Claim |
| + h := *want.Holder |
| + want.Holder = &h |
| + h.Admissions = append([]AdmissionRecord(nil), h.Admissions...) |
| + for i := 0; i < 3; i++ { |
| + h.Admissions[i].Covered = true |
| + } |
| + for repeat := 0; repeat < 2; repeat++ { |
| + o = updateGenerationTest(t, ctx, s, o, CoverParkedPredecessors(o.Claim.Op.ID, o.Claim.Op.Phase, "ie", []string{"G1", RootGeneration, "G1", "absent"})) |
| + want.Gen++ |
| + if !reflect.DeepEqual(o.Claim, &want) { |
| + t.Fatalf("sweep changed more than selected coverage: %+v", o.Claim) |
| + } |
| + } |
| + }) |
| + } |
| +} |
| + |
| +func TestCoverParkedPredecessorsRefusals(t *testing.T) { |
| + for _, name := range []string{"stale op", "stale phase", "nil holder", "empty region"} { |
| + t.Run(name, func(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + o := servingClaim(t, ctx, s) |
| + o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| + c.Op = Op{Kind: KindParking, ID: "sweep", Phase: "park"} |
| + c.Holder.Admissions = []AdmissionRecord{{Region: "ie", PredecessorRegion: "ie", Predecessor: "G1"}} |
| + if name == "nil holder" { |
| + c.Holder, c.Op = nil, Op{} |
| + } |
| + return nil |
| + }) |
| + op, phase, region := o.Claim.Op.ID, o.Claim.Op.Phase, "ie" |
| + switch name { |
| + case "stale op": |
| + op = "old" |
| + case "stale phase": |
| + phase = "old" |
| + case "empty region": |
| + region = "" |
| + } |
| + if _, err := Update(ctx, s, o, CoverParkedPredecessors(op, phase, region, []string{"G1"})); !errors.Is(err, ErrWrongState) { |
| + t.Fatalf("sweep = %v", err) |
| + } |
| + assertAdmissionStored(t, ctx, s, o) |
| + }) |
| + } |
| +} |
| + |
| +func TestCoverParkedPredecessorsComposesWithRetirement(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + o := servingClaim(t, ctx, s) |
| + cover := CoverParkedPredecessors("", "", "ie", []string{"G1"}) |
| + o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| + c.Holder.Admissions = []AdmissionRecord{{Region: "uk", PredecessorRegion: "ie", Predecessor: "G1"}} |
| + c.Obligations = append(c.Obligations, Obligation{ID: "park", Op: "sweep", Kind: "park-generation", Region: "ie", Generation: "G1", Retirement: "ret", Owner: "region:ie"}) |
| + return cover(c) |
| + }) |
| + if !o.Claim.Holder.Admissions[0].Covered { |
| + t.Fatal("queue CAS did not cover predecessor") |
| + } |
| + o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| + c.Holder.Admissions = append(c.Holder.Admissions, AdmissionRecord{Region: "uk", PredecessorRegion: "ie", Predecessor: "G1"}) |
| + return nil |
| + }) |
| + next, err := Update(ctx, s, o, func(c *Claim) error { |
| + if err := cover(c); err != nil { |
| + return err |
| + } |
| + return CompleteObligation("sweep", "park")(c) |
| + }) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + stored, err := Read(ctx, s, testHost) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if stored.ETag != next.ETag || len(stored.Claim.Obligations) != 0 || !stored.Claim.Holder.Admissions[1].Covered { |
| + t.Fatal("acknowledgement CAS did not cover later predecessor") |
| + } |
| +} |
| + |
| +func TestAdmitWriterAdditionalWritesLost(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + o := servingClaim(t, ctx, s) |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "ie", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "GS", PredecessorCertified: true})) |
| + cut := o.Claim.Holder.Session |
| + o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| + c.Op = Op{Kind: KindLanding, ID: "move", Phase: "probe", Exec: "boxT"} |
| + return nil |
| + }) |
| + var target []AdmissionRecord |
| + for _, mode := range []string{"cold", "warm", "warm"} { |
| + in := Admission{Region: "uk", ObservedOpID: "move", ObservedPhase: "probe", ObservedSession: o.Claim.Holder.Session, |
| + ObservedGeneration: o.Claim.Holder.Generation, Box: "boxT", Mode: mode, PredecessorCertified: true} |
| + if mode == "cold" { |
| + in.NewGeneration, in.Coverage = "GT", "sealed" |
| + } |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(in)) |
| + target = append(target, o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1]) |
| + } |
| + o = updateGenerationTest(t, ctx, s, o, func(c *Claim) error { |
| + c.Holder.Generation = "GS" |
| + c.Op = Op{Kind: KindReverting, ID: "move", Phase: "return", Exec: "boxS"} |
| + return nil |
| + }) |
| + additional := []WritesLost{} |
| + for i, record := range target[:2] { |
| + if record.Session <= cut { |
| + t.Fatal("target session does not follow the cut") |
| + } |
| + additional = append(additional, WritesLost{Box: record.Box, Session: record.Session, Generation: record.Generation, RemoteMaxTXID: uint64(10 + i)}) |
| + } |
| + wantAdditional := append([]WritesLost(nil), additional...) |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{ |
| + Region: "ie", ObservedOpID: "move", ObservedPhase: "return", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "GS", |
| + Box: "boxS", Mode: "cold", NewGeneration: "GS2", RemoteMaxTXID: 42, AdditionalWritesLost: additional, |
| + })) |
| + last := o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1] |
| + wantPrimary := &WritesLost{Box: "boxT", Session: target[2].Session, Generation: "GT", RemoteMaxTXID: 42} |
| + if !reflect.DeepEqual(last.WritesLost, wantPrimary) || !reflect.DeepEqual(last.AdditionalWritesLost, wantAdditional) { |
| + t.Fatalf("loss declarations = %+v, %+v", last.WritesLost, last.AdditionalWritesLost) |
| + } |
| + additional[0].Box = "changed" |
| + if !reflect.DeepEqual(last.AdditionalWritesLost, wantAdditional) { |
| + t.Fatal("record aliases the caller's loss declarations") |
| + } |
| + raw, err := json.Marshal(last) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var fields map[string]json.RawMessage |
| + if err := json.Unmarshal(raw, &fields); err != nil { |
| + t.Fatal(err) |
| + } |
| + var decoded []WritesLost |
| + if err := json.Unmarshal(fields["additional_writes_lost"], &decoded); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !reflect.DeepEqual(decoded, wantAdditional) { |
| + t.Fatalf("additional loss JSON = %s", raw) |
| + } |
| + assertAdmissionStored(t, ctx, s, o) |
| +} |
| + |
| +func TestAdmitWriterAdditionalWritesLostValidation(t *testing.T) { |
| + for _, name := range []string{"empty box", "zero session", "empty generation", "duplicate session", "duplicate predecessor", "same box different session", "same session different box", "root generation", "warm", "sealed"} { |
| + t.Run(name, func(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + o := servingClaim(t, ctx, s) |
| + o = updateGenerationTest(t, ctx, s, o, AdmitWriter(Admission{Region: "uk", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "GT"})) |
| + loss := WritesLost{Box: "boxT", Session: 42, Generation: "GT", RemoteMaxTXID: 0} |
| + in := Admission{Region: "ie", ObservedSession: o.Claim.Holder.Session, ObservedGeneration: "GT", Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: "GS"} |
| + switch name { |
| + case "empty box": |
| + loss.Box = "" |
| + case "zero session": |
| + loss.Session = 0 |
| + case "empty generation": |
| + loss.Generation = "" |
| + case "duplicate predecessor": |
| + loss.Box, loss.Session = o.Claim.Holder.WriterBox, o.Claim.Holder.Session |
| + case "root generation": |
| + loss.Generation = RootGeneration |
| + case "warm": |
| + in.Mode, in.NewGeneration, in.PredecessorCertified = "warm", "", true |
| + case "sealed": |
| + in.Coverage = "sealed" |
| + } |
| + in.AdditionalWritesLost = []WritesLost{loss} |
| + switch name { |
| + case "duplicate session", "same box different session", "same session different box": |
| + second := loss |
| + second.Generation, second.RemoteMaxTXID = "other", 99 |
| + if name == "same box different session" { |
| + second.Session++ |
| + } |
| + if name == "same session different box" { |
| + second.Box = "boxU" |
| + } |
| + in.AdditionalWritesLost = append(in.AdditionalWritesLost, second) |
| + } |
| + next, err := Update(ctx, s, o, AdmitWriter(in)) |
| + switch name { |
| + case "empty box", "zero session", "empty generation", "duplicate session", "duplicate predecessor": |
| + if !errors.Is(err, ErrWrongState) { |
| + t.Fatalf("invalid loss admission = %v", err) |
| + } |
| + assertAdmissionStored(t, ctx, s, o) |
| + default: |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + last := next.Claim.Holder.Admissions[len(next.Claim.Holder.Admissions)-1] |
| + if !reflect.DeepEqual(last.AdditionalWritesLost, in.AdditionalWritesLost) { |
| + t.Fatalf("loss declarations = %+v", last.AdditionalWritesLost) |
| + } |
| + assertAdmissionStored(t, ctx, s, next) |
| + } |
| + }) |
| + } |
| +} |